From 55fdf505dddb10bb4552763fd1daf699fd141fc0 Mon Sep 17 00:00:00 2001 From: john-sanchez31 Date: Mon, 17 Aug 2026 08:38:35 -0600 Subject: [PATCH 01/14] templates loading/calling --- .../collections/collection_metadata.py | 2 +- pydough/metadata/graphs/graph_metadata.py | 58 ++++++ pydough/metadata/parse.py | 161 ++++++++++++++++ pydough/metadata/templates/__init__.py | 7 + .../metadata/templates/attribute_metadata.py | 182 ++++++++++++++++++ pydough/unqualified/unqualified_node.py | 12 +- tests/test_metadata/sample_graphs.json | 77 ++++++++ tests/test_pipeline_tpch_custom.py | 38 ++++ 8 files changed, 535 insertions(+), 2 deletions(-) create mode 100644 pydough/metadata/templates/__init__.py create mode 100644 pydough/metadata/templates/attribute_metadata.py diff --git a/pydough/metadata/collections/collection_metadata.py b/pydough/metadata/collections/collection_metadata.py index a612d63e1..c3e50a6cf 100644 --- a/pydough/metadata/collections/collection_metadata.py +++ b/pydough/metadata/collections/collection_metadata.py @@ -1,5 +1,5 @@ """ -Base definition of PyDough metadaata for collections. +Base definition of PyDough metadata for collections. """ from abc import abstractmethod diff --git a/pydough/metadata/graphs/graph_metadata.py b/pydough/metadata/graphs/graph_metadata.py index a47b4eb5e..f132ff8d7 100644 --- a/pydough/metadata/graphs/graph_metadata.py +++ b/pydough/metadata/graphs/graph_metadata.py @@ -2,6 +2,7 @@ Definition of PyDough metadata for a graph. """ +from collections.abc import Callable from typing import TYPE_CHECKING from pydough.errors import PyDoughMetadataException @@ -26,6 +27,8 @@ class GraphMetadata(AbstractMetadata): "collections", "relationships", "functions", + "attributes", + "templates", "additional definitions", "verified pydough analysis", "extra semantic info", @@ -49,6 +52,8 @@ def __init__( self._name: str = name self._collections: dict[str, AbstractMetadata] = {} self._functions: dict[str, ExpressionFunctionOperator] = {} + self._attributes: dict[str, AbstractMetadata] = {} + self._templates: dict[str, Callable] = {} self._description = description self._synonyms = synonyms self._extra_semantic_info = extra_semantic_info @@ -74,6 +79,20 @@ def functions(self) -> dict[str, "ExpressionFunctionOperator"]: """ return self._functions + @property + def templates_attributes(self) -> dict[str, AbstractMetadata]: + """ + TODO + """ + return self._attributes + + @property + def templates_definitions(self) -> dict[str, Callable]: + """ + TODO + """ + return self._templates + @property def error_name(self) -> str: return f"graph {self.name!r}" @@ -193,3 +212,42 @@ def add_function(self, name: str, function: "ExpressionFunctionOperator") -> Non f"Function {name!r} already exists in {self.error_name}" ) self.functions[name] = function + + def add_template_attribute(self, new_attribute: AbstractMetadata) -> None: + """ + Adds a new collection to the graph. + + Args: + `collection`: the collection being inserted into the graph. + + Raises: + `PyDoughMetadataException`: if `collection` cannot be inserted + into the graph because. + """ + from pydough.metadata.templates import AttributeMetadata + + # Make sure the collection is actually a template_attribute + HasType(AttributeMetadata).verify(new_attribute, "attribute") + assert isinstance(new_attribute, AttributeMetadata) + + # Verify sure the attribute has not already been added to the graph + # and does not have a name collision with any other attributes in + # the graph. + if new_attribute.name in self._attributes: + if self.templates_attributes[new_attribute.name] == new_attribute: + raise PyDoughMetadataException( + f"Already added {new_attribute.error_name} to {self.error_name}" + ) + raise PyDoughMetadataException( + f"Duplicate attributes: {new_attribute.error_name} versus {self.templates_attributes[new_attribute.name].error_name}" + ) + self.templates_attributes[new_attribute.name] = new_attribute + + def add_template_definition(self, name: str, template: Callable) -> None: + """ + TODO + """ + # CHECK IF THERE IS A TEMPLATE WITH THE SAME NAME ALREADY + # CHECK IF THE NAME OF THE TEMPLATE CONFLICTS WIHT A BUILTIN PYTHON FUNCTION + + self.templates_definitions[name] = template diff --git a/pydough/metadata/parse.py b/pydough/metadata/parse.py index f80c3e75b..2f44f6ff1 100644 --- a/pydough/metadata/parse.py +++ b/pydough/metadata/parse.py @@ -4,7 +4,11 @@ __all__ = ["parse_json_metadata_from_file", "parse_metadata_from_list"] +import ast import json +import re +import textwrap +from collections.abc import Callable from typing import Any from pydough.errors import PyDoughMetadataException @@ -29,6 +33,7 @@ ReversiblePropertyMetadata, SimpleJoinMetadata, ) +from .templates import AttributeMetadata def parse_metadata(metadata_info: Any, graph_name: str) -> GraphMetadata | None: @@ -241,10 +246,46 @@ def parse_graph_v2(graph_name: str, graph_json: dict) -> GraphMetadata: assert isinstance(udf_definition, dict) parse_function_v2(graph, udf_definition) + if "templates" in graph_json: + templates_object: dict = extract_object( + graph_json, "templates", graph.error_name + ) + + # Attributes are optional + if "attributes" in templates_object: + attribute_definitions: list = extract_array( + templates_object, "attributes", graph.error_name + ) + for attribute_definition in attribute_definitions: + is_json_object.verify( + attribute_definition, + f"metadat for Templates definition inside {graph.error_name}", + ) + assert isinstance(attribute_definition, dict) + parse_template_attributes_v2(graph, attribute_definition) + else: + # TODO: Save it as an empty list [] + pass + + # Templates definitions + templates_definitions: list = extract_array( + templates_object, "definitions", graph.error_name + ) + + for template_definition in templates_definitions: + is_json_object.verify( + template_definition, + f"metadat for Templates definition inside {graph.error_name}", + ) + assert isinstance(template_definition, dict) + parse_template_definitions_v2(graph, template_definition) + NoExtraKeys(GraphMetadata.allowed_fields).verify(graph_json, graph.error_name) for collection in graph.collections.values(): assert isinstance(collection, CollectionMetadata) collection.verify_complete() + + # breakpoint() return graph @@ -504,3 +545,123 @@ def parse_function_v2(graph: GraphMetadata, udf_definition: dict) -> None: f"Unrecognized PyDough function type for {error_name}: {function_type!r}" ) graph.add_function(function_name, func) + + +def parse_template_attributes_v2(graph: GraphMetadata, attribute_json: dict) -> None: + """ + TODO + """ + attribute_name: str = extract_string( + attribute_json, "name", f"metadata for collections within {graph.error_name}" + ) + + AttributeMetadata.parse_from_json(graph, attribute_name, attribute_json) + + +def parse_template_definitions_v2(graph: GraphMetadata, definition_json: dict) -> None: + """ + TODO + Loads the template definition and save it in the graph property + """ + template_name: str = extract_string(definition_json, "name", graph.error_name) + answer_variable: str = extract_string( + definition_json, "answer_variable", graph.error_name + ) + code: str = extract_string(definition_json, "code", graph.error_name) + kwargs: dict[str, dict] = extract_object( + definition_json, "parameters", graph.error_name + ) + + # -------------------------------------------------------------------------- + arg_names: list[str] = list(kwargs.keys()) + args_parts: list[str] = [] + + part: str + for key, spec in kwargs.items(): + part = ( + f"{key}: {spec['type']}" + if "type" in spec and spec["type"] != "pydough" + else f"{key}" + ) + args_parts.append(part) + args: str = ", ".join(args_parts) + + # --- Replace {1}, {2}, ... with the corresponding argument name --- + def _replace_placeholder(match: re.Match) -> str: + index = int(match.group(1)) + if not (1 <= index <= len(arg_names)): + raise ValueError( + f"Placeholder {{{index}}} in pydough_code has no matching " + f"argument (only {len(arg_names)} args provided)." + ) + return arg_names[index - 1] + + substituted_code = re.sub(r"\{(\d+)\}", _replace_placeholder, code) + + # --- Indent and assemble the final template --- + indented_code = textwrap.indent(substituted_code.strip(), " ") + + template_str: str = ( + f"def {template_name}({args}):\n{indented_code}\n return {answer_variable}\n" + ) + + loaded_template: Callable = build_function_from_string( + template_str, + arg_names, + template_name, + graph, + ) + + graph.add_template_definition(template_name, loaded_template) + + +def build_function_from_string( + template_str: str, + args: list[str], + template_name: str, + metadata: GraphMetadata | None = None, +) -> Callable: + """ + Builds a callable from a PyDough source string, without executing it. + Intended for constructing template functions that get stored (e.g. via + `graph.add_template_definition`) and invoked later. + + Args: + `template_str`: the PyDough code that forms the body of the function. + `args`: the parameter names of the generated function. + `template_name`: the name to give the generated function. + `answer_variable`: the variable in `source` holding the return value. + Defaults to "result". + `metadata`: the metadata graph bound into the function's closure as + `_graph`. Defaults to `pydough.active_session.metadata`. + `environment`: extra names available both when transforming the + source and inside the function's closure. + + Returns: + The callable built from `source`, not yet invoked. + """ + import pydough + from pydough.unqualified.unqualified_transform import AddRootVisitor + + # Args + "pydough" are "known" so they aren't + # rewritten into _ROOT. by the visitor. + known_names: set[str] = set(args) | {"pydough"} + graph_name: str = "_graph" + visitor = AddRootVisitor(graph_name, known_names) + + tree: ast.AST = ast.parse(template_str) + new_tree: ast.AST = ast.fix_missing_locations(visitor.visit(tree)) + transformed_code: str = ast.unparse(new_tree) + + compiled = compile(transformed_code, filename=f"<{template_name}>", mode="exec") + + # `_graph` and `pydough` are baked into the function's globals so that + # they're available whenever the function is later called + namespace: dict[str, Any] = { + graph_name: metadata, + "pydough": pydough, + } + exec(compiled, namespace, namespace) + + loaded_template: Callable = namespace[template_name] + return loaded_template diff --git a/pydough/metadata/templates/__init__.py b/pydough/metadata/templates/__init__.py new file mode 100644 index 000000000..a10449f42 --- /dev/null +++ b/pydough/metadata/templates/__init__.py @@ -0,0 +1,7 @@ +""" +Submodule of the PyDough metadata module defining metadata for templates. +""" + +__all__ = ["AttributeMetadata"] + +from .attribute_metadata import AttributeMetadata diff --git a/pydough/metadata/templates/attribute_metadata.py b/pydough/metadata/templates/attribute_metadata.py new file mode 100644 index 000000000..27e5b7122 --- /dev/null +++ b/pydough/metadata/templates/attribute_metadata.py @@ -0,0 +1,182 @@ +""" +Base definition of PyDough metadata for template attributes. +""" + +from pydough.errors.error_utils import ( + HasType, + extract_array, + extract_integer, + extract_string, +) +from pydough.metadata.abstract_metadata import AbstractMetadata +from pydough.metadata.graphs import GraphMetadata + + +class AttributeMetadata(AbstractMetadata): + """ + Abstract base class for PyDough metadata for template attributes. + """ + + # Set of names of fields that can be included in the JSON + # object describing a template attribute. Implementations should extend this. + allowed_fields: set[str] = { + "name", + "usage", + "type", + "description", + "options", + } + + def __init__( + self, + name: str, + graph: GraphMetadata, + usage: list[str], + type: str, + description: str, + ): + # TODO: Check if the name is a valid one (not a the graph, collection name) + # Really this name is not going to be use for other than just identifying + # the attribute so maybe is not really worth the check + HasType(GraphMetadata).verify(graph, f"graph {name!r}") + + self._graph: GraphMetadata = graph + self._name: str = name + self._usage: list[str] = usage + self._type: str = type + self._options: dict[str, str | int] = {} + + super().__init__(description, None, None) + + @property + def graph(self) -> GraphMetadata: + """ + The graph that the template attribute belongs to. + """ + return self._graph + + @property + def name(self) -> str: + """ + The name of the template attribute + """ + return self._name + + @property + def usage(self) -> list[str]: + """ + List with the names of the templates where the attribute can be used + """ + return self._usage + + @property + def type(self) -> str: + """ + Type of the data saved on the options for the attribute. + + NOTE: `'pydough'` is a special type to identify if the value is a pydough + expression. + """ + + return self._type + + @property + def options(self) -> dict[str, str | int]: + """ + List with all options of the attribute + """ + return self._options + + @property + def error_name(self): + return self.create_error_name(self.name, self.graph.error_name) + + @property + def components(self): + comp: list = [self.name, self.description, self.type] + comp.extend(self.usage) + return comp + + @property + def path(self) -> str: + return f"{self.graph.path}.{self.name}" + + def add_attribute_option(self, label: str, value: str | int) -> None: + """ + Add an option to the list of options + """ + if label in self.options: + raise ValueError(f"Duplicate option label: {label!r}") + + self.options[label] = value + + @staticmethod + def create_error_name(name: str, graph_error_name: str): + return f"template attribute {name!r} in {graph_error_name}" + + def verify_complete(self) -> None: + """ + Verifies that a template attribute is well-formed after the parsing of all of + its properties is complete. Subclasses should extend the checks done + in the default implementation. + + Raises: + `PyDoughMetadataException`: if the template attribute is malformed + in any way after parsing is done. + """ + # TODO + return + + @staticmethod + def parse_from_json( + graph: GraphMetadata, attribute_name: str, attribute_json: dict + ) -> None: + """ + Parses a JSON object into the metadata for a template attribute + and inserts it into the graph. + + Args: + `graph`: the metadata for the graph that the template attribute will + be added to. + `attribute_name`: the name of the template attribute that will be + added to the graph. + `attribute_json`: the JSON object that is being parsed to create + the new template attribute. + + Raises: + `PyDoughMetadataException`: if the JSON does not meet the necessary + structure properties. + """ + + error_name: str = AttributeMetadata.create_error_name( + attribute_name, graph.error_name + ) + + # Extract the relevant properties from the JSON to build the new template + # attribute, then add it to the graph + attr_usage: list[str] = extract_array(attribute_json, "usage", error_name) + attr_type: str = extract_string(attribute_json, "type", error_name) + attr_desc: str = extract_string(attribute_json, "description", error_name) + + new_attribute: AttributeMetadata = AttributeMetadata( + attribute_name, + graph, + attr_usage, + attr_type, + attr_desc, + ) + + # Parse and add the options + attr_options: list = extract_array(attribute_json, "options", error_name) + + for option in attr_options: + label: str = extract_string(option, "label", error_name) + value: str | int + try: + value = extract_string(option, "value", error_name) + except AssertionError: + value = extract_integer(option, "value", error_name) + + new_attribute.add_attribute_option(label, value) + + graph.add_template_attribute(new_attribute) diff --git a/pydough/unqualified/unqualified_node.py b/pydough/unqualified/unqualified_node.py index c0f693cb6..0186af36e 100644 --- a/pydough/unqualified/unqualified_node.py +++ b/pydough/unqualified/unqualified_node.py @@ -23,7 +23,7 @@ ] from abc import ABC -from collections.abc import Iterable +from collections.abc import Callable, Iterable from datetime import date, datetime from typing import Any, Union @@ -134,6 +134,16 @@ def __getitem__(self, key): ) def __call__(self, *args, **kwargs): + + if pydough.active_session.metadata: + metadata_templates: dict[str, Callable] = ( + pydough.active_session.metadata.templates_definitions + ) + + name = str(self) + if name in metadata_templates: + return metadata_templates[name](*args, **kwargs) + raise pydough.active_session.error_builder.undefined_function_call( self, *args, **kwargs ) diff --git a/tests/test_metadata/sample_graphs.json b/tests/test_metadata/sample_graphs.json index 9f99dc3d6..15bd041ba 100644 --- a/tests/test_metadata/sample_graphs.json +++ b/tests/test_metadata/sample_graphs.json @@ -845,6 +845,83 @@ "synonyms": ["transactions", "purchases"] } ], + "templates": { + "attributes": [ + { + "name": "years", + "usage": [], + "type": "int", + "description": "The year for which to calculate the revenue of orders.", + "options": [ + { "label": "Year 1992", "value": 1992}, + { "label": "Year 1993", "value": 1993}, + { "label": "Year 1994", "value": 1994}, + { "label": "Year 1995", "value": 1995}, + { "label": "Year 1996", "value": 1996}, + { "label": "Year 1997", "value": 1997}, + { "label": "Year 1998", "value": 1998} + ] + }, + { + "name": "order_dimensions", + "usage": ["orders_revenue_by"], + "type": "pydough", + "description": "The dimension by which to partition the orders calculation.", + "options": [ + {"value": "customer.market_segment", "label": "Customer Market Segment"}, + {"value": "customer.nation.name", "label": "Customer Nation"}, + {"value": "customer.nation.region.name", "label": "Customer Region"}, + {"value": "order_priority", "label": "Order Priority"}, + {"value": "MONTHNAME(order_date)", "label": "Month"}, + {"value": "clerk", "label": "Clerk"} + ] + } + ], + "definitions": [ + { + "name": "order_revenue", + "dependencies": [], + "description": "Calculates the revenue of an order, which is the sum of the extended price * (1 - discount) for all line items in the order.", + "parameters": {}, + "code": "result = SUM(lines.extended_price * (1 - lines.discount))\n", + "answer_variable": "result" + }, + { + "name": "orders_revenue_by", + "description": "Calculates the revenue of orders in a given year, partitioned by a specified dimension.", + "dependencies": ["order_revenue"], + "parameters": { + "arg_year": { + "type": "int", + "description": "The year for which to calculate the revenue of orders." + }, + "arg_dimension": { + "type": "pydough", + "description": "The dimension by which to partition the revenue calculation." + } + }, + "code": "result = orders.WHERE(({1} == YEAR(order_date))).CALCULATE(revenue=order_revenue(), dimension=({2})).PARTITION(name=\"orders_groups\", by=dimension).CALCULATE(dimension, segment_revenue=SUM(orders.revenue))\n", + "answer_variable": "result" + }, + { + "name": "top_bottom_comparison", + "description": "Compares the top and bottom groups of a partitioned orders", + "dependencies": [], + "parameters": { + "arg_partitioned_orders": { + "type": "pydough", + "description": "The partitioned orders to compare." + }, + "arg_calculation": { + "type": "pydough", + "description": "The metric by which to compare the groups." + } + }, + "code": "result = {1}.CALCULATE(dimension, segment_revenue, comparison_value={2}).WHERE(ABSENT(PREV(segment_revenue, by=segment_revenue.DESC())) | ABSENT(NEXT(segment_revenue, by=segment_revenue.DESC())) )", + "answer_variable": "result" + } + ] + }, "additional definitions": [ "Revenue for a lineitem is the extended_price * (1 - discount) * (1 - tax) minus quantity * supply_cost from the corresponding supply record", "A domestic shipment is a lineitem where the customer and supplier are from the same nation", diff --git a/tests/test_pipeline_tpch_custom.py b/tests/test_pipeline_tpch_custom.py index 1b6410dcc..6e46008d3 100644 --- a/tests/test_pipeline_tpch_custom.py +++ b/tests/test_pipeline_tpch_custom.py @@ -5208,6 +5208,44 @@ ), id="monthname_function_1", ), + pytest.param( + PyDoughPandasTest( + "result = orders.WHERE(" + "(1996 == YEAR(order_date))\n" + ").CALCULATE(\n" + "revenue=order_revenue()\n" + ")\n", + "TPCH", + lambda: pd.DataFrame({}), + "template_test", + ), + id="template_test", + ), + pytest.param( + PyDoughPandasTest( + "result = orders_revenue_by(1996, customer.market_segment).WHERE(" + "(dimension == 'FURNITURE')\n" + ")\n", + "TPCH", + lambda: pd.DataFrame( + {"dimension": ["FURNITURE"], "segment_revenue": [6.671056e09]} + ), + "nested_template", + ), + id="nested_template", + ), + pytest.param( + PyDoughPandasTest( + "selected_customers = customers.WHERE(" + "(account_balance >= template_literal(9000))\n" + ")\n" + "result = TPCH.CALCULATE(n_custs=COUNT(selected_customers))\n", + "TPCH", + lambda: pd.DataFrame({"n_custs": [13533]}), + "literal_template", + ), + id="literal_template", + ), ], ) def tpch_custom_pipeline_test_data(request) -> PyDoughPandasTest: From 3a6767273b188946af0fbe97f8040f7e086e3a6e Mon Sep 17 00:00:00 2001 From: john-sanchez31 Date: Wed, 19 Aug 2026 15:08:05 -0600 Subject: [PATCH 02/14] adding template metadata class --- pydough/metadata/graphs/graph_metadata.py | 72 +++-- pydough/metadata/parse.py | 136 ++------- pydough/metadata/templates/__init__.py | 3 +- .../metadata/templates/attribute_metadata.py | 44 ++- .../metadata/templates/template_metadata.py | 274 ++++++++++++++++++ pydough/unqualified/unqualified_node.py | 5 +- 6 files changed, 374 insertions(+), 160 deletions(-) create mode 100644 pydough/metadata/templates/template_metadata.py diff --git a/pydough/metadata/graphs/graph_metadata.py b/pydough/metadata/graphs/graph_metadata.py index f132ff8d7..afff19001 100644 --- a/pydough/metadata/graphs/graph_metadata.py +++ b/pydough/metadata/graphs/graph_metadata.py @@ -2,7 +2,6 @@ Definition of PyDough metadata for a graph. """ -from collections.abc import Callable from typing import TYPE_CHECKING from pydough.errors import PyDoughMetadataException @@ -10,6 +9,7 @@ from pydough.metadata.abstract_metadata import AbstractMetadata if TYPE_CHECKING: + from pydough.metadata.templates import AttributeMetadata, TemplateMetadata from pydough.pydough_operators import ( ExpressionFunctionOperator, ) @@ -52,8 +52,8 @@ def __init__( self._name: str = name self._collections: dict[str, AbstractMetadata] = {} self._functions: dict[str, ExpressionFunctionOperator] = {} - self._attributes: dict[str, AbstractMetadata] = {} - self._templates: dict[str, Callable] = {} + self._attributes: dict[str, AttributeMetadata] = {} + self._templates: dict[str, TemplateMetadata] = {} self._description = description self._synonyms = synonyms self._extra_semantic_info = extra_semantic_info @@ -80,16 +80,16 @@ def functions(self) -> dict[str, "ExpressionFunctionOperator"]: return self._functions @property - def templates_attributes(self) -> dict[str, AbstractMetadata]: + def templates_attributes(self) -> dict[str, "AttributeMetadata"]: """ - TODO + The attributes that can be used with the templates within the graph. """ return self._attributes @property - def templates_definitions(self) -> dict[str, Callable]: + def templates_definitions(self) -> dict[str, "TemplateMetadata"]: """ - TODO + The templates defined within the graph. """ return self._templates @@ -198,6 +198,9 @@ def add_function(self, name: str, function: "ExpressionFunctionOperator") -> Non `PyDoughMetadataException`: if `function` cannot be inserted into the graph because of a name collision. """ + # Cirular import error raises if the import is made globally + from pydough.pydough_operators import builtin_registered_operators + is_valid_name.verify(name, f"function name {name!r}") if name == self.name: raise PyDoughMetadataException( @@ -207,6 +210,10 @@ def add_function(self, name: str, function: "ExpressionFunctionOperator") -> Non raise PyDoughMetadataException( f"Function name {name!r} cannot be the same as a collection name in {self.error_name}" ) + if name in builtin_registered_operators(): + raise PyDoughMetadataException( + f"Function name {name!r} already in use for a PyDough operador" + ) if name in self.functions: raise PyDoughMetadataException( f"Function {name!r} already exists in {self.error_name}" @@ -215,25 +222,25 @@ def add_function(self, name: str, function: "ExpressionFunctionOperator") -> Non def add_template_attribute(self, new_attribute: AbstractMetadata) -> None: """ - Adds a new collection to the graph. + Adds a new attribute to the graph. Args: - `collection`: the collection being inserted into the graph. + `new_attribute`: the attribute being inserted into the graph. Raises: - `PyDoughMetadataException`: if `collection` cannot be inserted + `PyDoughMetadataException`: if `new_attribute` cannot be inserted into the graph because. """ from pydough.metadata.templates import AttributeMetadata - # Make sure the collection is actually a template_attribute + # Make sure the new_attribute is actually a template_attribute HasType(AttributeMetadata).verify(new_attribute, "attribute") assert isinstance(new_attribute, AttributeMetadata) - # Verify sure the attribute has not already been added to the graph + # Verify the attribute has not already been added to the graph # and does not have a name collision with any other attributes in # the graph. - if new_attribute.name in self._attributes: + if new_attribute.name in self.templates_attributes: if self.templates_attributes[new_attribute.name] == new_attribute: raise PyDoughMetadataException( f"Already added {new_attribute.error_name} to {self.error_name}" @@ -243,11 +250,40 @@ def add_template_attribute(self, new_attribute: AbstractMetadata) -> None: ) self.templates_attributes[new_attribute.name] = new_attribute - def add_template_definition(self, name: str, template: Callable) -> None: + def add_template_definition( + self, name: str, new_template: AbstractMetadata + ) -> None: """ - TODO + Adds a new template to the graph. + + Args: + `name`: the name of the new template being added. + `template`: the template callable being inserted into the graph + + Raises: + `PyDoughMetadataException`: if `template` cannot be inserted + into the graph because. """ - # CHECK IF THERE IS A TEMPLATE WITH THE SAME NAME ALREADY - # CHECK IF THE NAME OF THE TEMPLATE CONFLICTS WIHT A BUILTIN PYTHON FUNCTION + from pydough.metadata.templates import TemplateMetadata + + # Cirular import error raises if the import is made globally + from pydough.pydough_operators import builtin_registered_operators + + # Make sure the new_template is actually a template_attribute + HasType(TemplateMetadata).verify(new_template, "template") + assert isinstance(new_template, TemplateMetadata) + + if name in self.templates_definitions: + if self.templates_definitions[name] == new_template: + raise PyDoughMetadataException( + f"Already added {name} to {self.error_name}" + ) + raise PyDoughMetadataException( + f"Duplicate templates: {name} versus {self.templates_definitions[name]}" + ) + if name in builtin_registered_operators(): + raise PyDoughMetadataException( + f"The template '{name}' already exists as a PyDough operador" + ) - self.templates_definitions[name] = template + self.templates_definitions[name] = new_template diff --git a/pydough/metadata/parse.py b/pydough/metadata/parse.py index 2f44f6ff1..68bc6de00 100644 --- a/pydough/metadata/parse.py +++ b/pydough/metadata/parse.py @@ -4,11 +4,7 @@ __all__ = ["parse_json_metadata_from_file", "parse_metadata_from_list"] -import ast import json -import re -import textwrap -from collections.abc import Callable from typing import Any from pydough.errors import PyDoughMetadataException @@ -33,7 +29,7 @@ ReversiblePropertyMetadata, SimpleJoinMetadata, ) -from .templates import AttributeMetadata +from .templates import AttributeMetadata, TemplateMetadata def parse_metadata(metadata_info: Any, graph_name: str) -> GraphMetadata | None: @@ -263,9 +259,6 @@ def parse_graph_v2(graph_name: str, graph_json: dict) -> GraphMetadata: ) assert isinstance(attribute_definition, dict) parse_template_attributes_v2(graph, attribute_definition) - else: - # TODO: Save it as an empty list [] - pass # Templates definitions templates_definitions: list = extract_array( @@ -278,14 +271,13 @@ def parse_graph_v2(graph_name: str, graph_json: dict) -> GraphMetadata: f"metadat for Templates definition inside {graph.error_name}", ) assert isinstance(template_definition, dict) - parse_template_definitions_v2(graph, template_definition) + parse_template_definition_v2(graph, template_definition) NoExtraKeys(GraphMetadata.allowed_fields).verify(graph_json, graph.error_name) for collection in graph.collections.values(): assert isinstance(collection, CollectionMetadata) collection.verify_complete() - # breakpoint() return graph @@ -549,119 +541,37 @@ def parse_function_v2(graph: GraphMetadata, udf_definition: dict) -> None: def parse_template_attributes_v2(graph: GraphMetadata, attribute_json: dict) -> None: """ - TODO + Parses the JSON object for a PyDough template attribute in version 2 of the + PyDough metadata format. + + Args: + `graph`: the metadata for the graph that the attribute would be + added to. The attribute will be added to this graph in-place. + `attribute_json`: the JSON object containing the metadata for the + template attribute. + + Raises: + `PyDoughMetadataException`: if the JSON does not meet the necessary + structure properties. """ attribute_name: str = extract_string( - attribute_json, "name", f"metadata for collections within {graph.error_name}" + attribute_json, + "name", + f"metadata for template attribute within {graph.error_name}", ) AttributeMetadata.parse_from_json(graph, attribute_name, attribute_json) -def parse_template_definitions_v2(graph: GraphMetadata, definition_json: dict) -> None: +def parse_template_definition_v2(graph: GraphMetadata, definition_json: dict) -> None: """ TODO Loads the template definition and save it in the graph property """ - template_name: str = extract_string(definition_json, "name", graph.error_name) - answer_variable: str = extract_string( - definition_json, "answer_variable", graph.error_name - ) - code: str = extract_string(definition_json, "code", graph.error_name) - kwargs: dict[str, dict] = extract_object( - definition_json, "parameters", graph.error_name - ) - - # -------------------------------------------------------------------------- - arg_names: list[str] = list(kwargs.keys()) - args_parts: list[str] = [] - - part: str - for key, spec in kwargs.items(): - part = ( - f"{key}: {spec['type']}" - if "type" in spec and spec["type"] != "pydough" - else f"{key}" - ) - args_parts.append(part) - args: str = ", ".join(args_parts) - - # --- Replace {1}, {2}, ... with the corresponding argument name --- - def _replace_placeholder(match: re.Match) -> str: - index = int(match.group(1)) - if not (1 <= index <= len(arg_names)): - raise ValueError( - f"Placeholder {{{index}}} in pydough_code has no matching " - f"argument (only {len(arg_names)} args provided)." - ) - return arg_names[index - 1] - - substituted_code = re.sub(r"\{(\d+)\}", _replace_placeholder, code) - - # --- Indent and assemble the final template --- - indented_code = textwrap.indent(substituted_code.strip(), " ") - - template_str: str = ( - f"def {template_name}({args}):\n{indented_code}\n return {answer_variable}\n" - ) - - loaded_template: Callable = build_function_from_string( - template_str, - arg_names, - template_name, - graph, + template_name: str = extract_string( + definition_json, + "name", + f"metadata for template definition within {graph.error_name}", ) - graph.add_template_definition(template_name, loaded_template) - - -def build_function_from_string( - template_str: str, - args: list[str], - template_name: str, - metadata: GraphMetadata | None = None, -) -> Callable: - """ - Builds a callable from a PyDough source string, without executing it. - Intended for constructing template functions that get stored (e.g. via - `graph.add_template_definition`) and invoked later. - - Args: - `template_str`: the PyDough code that forms the body of the function. - `args`: the parameter names of the generated function. - `template_name`: the name to give the generated function. - `answer_variable`: the variable in `source` holding the return value. - Defaults to "result". - `metadata`: the metadata graph bound into the function's closure as - `_graph`. Defaults to `pydough.active_session.metadata`. - `environment`: extra names available both when transforming the - source and inside the function's closure. - - Returns: - The callable built from `source`, not yet invoked. - """ - import pydough - from pydough.unqualified.unqualified_transform import AddRootVisitor - - # Args + "pydough" are "known" so they aren't - # rewritten into _ROOT. by the visitor. - known_names: set[str] = set(args) | {"pydough"} - graph_name: str = "_graph" - visitor = AddRootVisitor(graph_name, known_names) - - tree: ast.AST = ast.parse(template_str) - new_tree: ast.AST = ast.fix_missing_locations(visitor.visit(tree)) - transformed_code: str = ast.unparse(new_tree) - - compiled = compile(transformed_code, filename=f"<{template_name}>", mode="exec") - - # `_graph` and `pydough` are baked into the function's globals so that - # they're available whenever the function is later called - namespace: dict[str, Any] = { - graph_name: metadata, - "pydough": pydough, - } - exec(compiled, namespace, namespace) - - loaded_template: Callable = namespace[template_name] - return loaded_template + TemplateMetadata.parse_from_json(graph, template_name, definition_json) diff --git a/pydough/metadata/templates/__init__.py b/pydough/metadata/templates/__init__.py index a10449f42..79f857393 100644 --- a/pydough/metadata/templates/__init__.py +++ b/pydough/metadata/templates/__init__.py @@ -2,6 +2,7 @@ Submodule of the PyDough metadata module defining metadata for templates. """ -__all__ = ["AttributeMetadata"] +__all__ = ["AttributeMetadata", "TemplateMetadata"] from .attribute_metadata import AttributeMetadata +from .template_metadata import TemplateMetadata diff --git a/pydough/metadata/templates/attribute_metadata.py b/pydough/metadata/templates/attribute_metadata.py index 27e5b7122..ad01f94c7 100644 --- a/pydough/metadata/templates/attribute_metadata.py +++ b/pydough/metadata/templates/attribute_metadata.py @@ -2,6 +2,7 @@ Base definition of PyDough metadata for template attributes. """ +from pydough.errors.error_types import PyDoughMetadataException from pydough.errors.error_utils import ( HasType, extract_array, @@ -14,11 +15,14 @@ class AttributeMetadata(AbstractMetadata): """ - Abstract base class for PyDough metadata for template attributes. + Concrete metadata implementation class for PyDough template attributes. + Representing the options (labels and values) that can be used on the templates + definitions. """ # Set of names of fields that can be included in the JSON - # object describing a template attribute. Implementations should extend this. + # object describing a template attribute. + # TODO: Maybe create a verify complete function with this if not delete it allowed_fields: set[str] = { "name", "usage", @@ -35,9 +39,6 @@ def __init__( type: str, description: str, ): - # TODO: Check if the name is a valid one (not a the graph, collection name) - # Really this name is not going to be use for other than just identifying - # the attribute so maybe is not really worth the check HasType(GraphMetadata).verify(graph, f"graph {name!r}") self._graph: GraphMetadata = graph @@ -99,7 +100,11 @@ def components(self): @property def path(self) -> str: - return f"{self.graph.path}.{self.name}" + return f"{self.graph.path}.templates.attributes.{self.name}" + + @staticmethod + def create_error_name(name: str, graph_error_name: str): + return f"template attribute {name!r} in {graph_error_name}" def add_attribute_option(self, label: str, value: str | int) -> None: """ @@ -110,23 +115,6 @@ def add_attribute_option(self, label: str, value: str | int) -> None: self.options[label] = value - @staticmethod - def create_error_name(name: str, graph_error_name: str): - return f"template attribute {name!r} in {graph_error_name}" - - def verify_complete(self) -> None: - """ - Verifies that a template attribute is well-formed after the parsing of all of - its properties is complete. Subclasses should extend the checks done - in the default implementation. - - Raises: - `PyDoughMetadataException`: if the template attribute is malformed - in any way after parsing is done. - """ - # TODO - return - @staticmethod def parse_from_json( graph: GraphMetadata, attribute_name: str, attribute_json: dict @@ -172,10 +160,14 @@ def parse_from_json( for option in attr_options: label: str = extract_string(option, "label", error_name) value: str | int + try: - value = extract_string(option, "value", error_name) - except AssertionError: - value = extract_integer(option, "value", error_name) + if type(option["value"]) is str: + value = extract_string(option, "value", error_name) + else: + value = extract_integer(option, "value", error_name) + except PyDoughMetadataException: + raise PyDoughMetadataException("Option value must be string or integer") new_attribute.add_attribute_option(label, value) diff --git a/pydough/metadata/templates/template_metadata.py b/pydough/metadata/templates/template_metadata.py new file mode 100644 index 000000000..f3b1b25b0 --- /dev/null +++ b/pydough/metadata/templates/template_metadata.py @@ -0,0 +1,274 @@ +""" +Base definition of PyDough metadata for template definition. +""" + +import ast +import re +import textwrap +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any + +from pydough.errors.error_types import PyDoughMetadataException +from pydough.errors.error_utils import extract_object, extract_string +from pydough.metadata.abstract_metadata import AbstractMetadata +from pydough.metadata.graphs.graph_metadata import GraphMetadata + + +@dataclass +class TemplateParameter: + """ + TODO + """ + + name: str + """ + Name of the parameter + """ + + type: str + """ + Type of data this parameter receives. + NOTE: `pydough` type will keep this parameter without type hint. + """ + + description: str + """ + Detail of what this parameter represents inside the template or how it is used + """ + + +class TemplateMetadata(AbstractMetadata): + """ + Concrete metadata implementation class for PyDough template definitions. + """ + + def __init__( + self, + name: str, + graph: GraphMetadata, + description: str, + parameters: dict[str, dict[str, str]], + code: str, + answer_variable: str = "result", + ): + self._graph: GraphMetadata = graph + self._name: str = name + self._code: str = code + self._answer_variable: str = answer_variable + + self._parameters: dict[str, TemplateParameter] = ( + TemplateMetadata.parse_parameters_from_json(parameters) + ) + self._template_callable: Callable = self.build_template_callable(graph) + + super().__init__(description, None, None) + + @property + def graph(self) -> GraphMetadata: + """ + TODO + """ + return self._graph + + @property + def name(self) -> str: + """ + TODO + """ + return self._name + + @property + def parameters(self) -> dict[str, TemplateParameter]: + """ + TODO + """ + return self._parameters + + @property + def code(self) -> str: + """ + TODO + """ + return self._code + + @property + def answer_variable(self) -> str: + """ + TODO + """ + return self._answer_variable + + @property + def template_callable(self) -> Callable | None: + """ + TODO + """ + return self._template_callable + + @property + def error_name(self): + return self.create_error_name(self.name, self.graph.error_name) + + @property + def components(self): + comp: list = [self.name, self.description, self.code] + return comp + + @property + def path(self) -> str: + # TODO: Not sure about the usage of this path + return f"{self.graph.path}.templates.definitions.{self.name}" + + def __call__(self, *args, **kwds): + if self.template_callable is None: + raise ValueError("") + return ( + self.template_callable(*args, **kwds) + if self.template_callable is not None + else None + ) + + @staticmethod + def create_error_name(name: str, graph_error_name: str): + return f"template definition {name!r} in {graph_error_name}" + + @staticmethod + def parse_from_json(graph: GraphMetadata, name: str, definition_json: dict) -> None: + """ + TODO + """ + description: str = extract_string( + definition_json, "description", graph.error_name + ) + answer_variable: str = extract_string( + definition_json, "answer_variable", graph.error_name + ) + code: str = extract_string(definition_json, "code", graph.error_name) + kwargs: dict[str, dict] = extract_object( + definition_json, "parameters", graph.error_name + ) + + new_template: TemplateMetadata = TemplateMetadata( + name, + graph, + description, + kwargs, + code, + answer_variable, + ) + + graph.add_template_definition(name, new_template) + + @staticmethod + def parse_parameters_from_json( + parameters_json: dict, + ) -> dict[str, TemplateParameter]: + """ + TODO + """ + + template_params: dict[str, TemplateParameter] = {} + + for param_name, arg in parameters_json.items(): + if param_name in template_params: + raise PyDoughMetadataException( + f"Already added {param_name} to the template's parameters" + ) + + param_type: str = extract_string( + arg, "type", "All parameters must have type" + ) + param_description: str = extract_string( + arg, "description", "All parameters must have description" + ) + new_param = TemplateParameter(param_name, param_type, param_description) + + template_params[param_name] = new_param + + return template_params + + def build_template_callable( + self, + metadata: GraphMetadata | None = None, + ) -> Callable: + """ + Builds a callable from a PyDough source string, without executing it. + Intended for constructing template functions that get stored (e.g. via + `graph.add_template_definition`) and invoked later. + + Args: + `template_str`: the PyDough code that forms the body of the function. + `args`: the parameter names of the generated function. + `template_name`: the name to give the generated function. + `answer_variable`: the variable in `source` holding the return value. + Defaults to "result". + `metadata`: the metadata graph bound into the function's closure as + `_graph`. Defaults to `pydough.active_session.metadata`. + `environment`: extra names available both when transforming the + source and inside the function's closure. + + Returns: + The callable built from `source`, not yet invoked. + """ + + template_str: str = self.generate_template_str() + + import pydough + from pydough.unqualified.unqualified_transform import AddRootVisitor + + # Args + "pydough" are "known" so they aren't + # rewritten into _ROOT. by the visitor. + known_names: set[str] = {"pydough"} + graph_name: str = "_graph" + visitor = AddRootVisitor(graph_name, known_names) + + tree: ast.AST = ast.parse(template_str) + new_tree: ast.AST = ast.fix_missing_locations(visitor.visit(tree)) + transformed_code: str = ast.unparse(new_tree) + + compiled = compile(transformed_code, filename=f"<{self.name}>", mode="exec") + + # `_graph` and `pydough` are baked into the function's globals so that + # they're available whenever the function is later called + namespace: dict[str, Any] = { + graph_name: metadata, + "pydough": pydough, + } + exec(compiled, namespace, namespace) + + loaded_template: Callable = namespace[self.name] + return loaded_template + + def generate_template_str(self) -> str: + """ + TODO + """ + # Parsing arguments + arg_names: list[str] = list(self.parameters.keys()) + args_parts: list[str] = [] + + part: str + for key, spec in self.parameters.items(): + part = f"{key}: {spec.type}" if spec.type != "pydough" else f"{key}" + args_parts.append(part) + template_args: str = ", ".join(args_parts) + + # --- Replace {1}, {2}, ... with the corresponding argument name --- + def _replace_placeholder(match: re.Match) -> str: + index = int(match.group(1)) + if not (1 <= index <= len(arg_names)): + raise ValueError( + f"Placeholder {{{index}}} in pydough_code has no matching " + f"argument (only {len(arg_names)} args provided)." + ) + return arg_names[index - 1] + + substituted_code = re.sub(r"\{(\d+)\}", _replace_placeholder, self.code) + + # --- Indent and assemble the final template --- + indented_code: str = textwrap.indent(substituted_code.strip(), " ") + + template_str: str = f"def {self.name}({template_args}):\n{indented_code}\n return {self.answer_variable}\n" + + return template_str diff --git a/pydough/unqualified/unqualified_node.py b/pydough/unqualified/unqualified_node.py index 0186af36e..a73e43891 100644 --- a/pydough/unqualified/unqualified_node.py +++ b/pydough/unqualified/unqualified_node.py @@ -23,7 +23,7 @@ ] from abc import ABC -from collections.abc import Callable, Iterable +from collections.abc import Iterable from datetime import date, datetime from typing import Any, Union @@ -32,6 +32,7 @@ from pydough.errors import PyDoughUnqualifiedException from pydough.errors.error_utils import is_bool, is_integer, is_positive_int, is_string from pydough.metadata import GraphMetadata +from pydough.metadata.templates import TemplateMetadata from pydough.types import ( ArrayType, BooleanType, @@ -136,7 +137,7 @@ def __getitem__(self, key): def __call__(self, *args, **kwargs): if pydough.active_session.metadata: - metadata_templates: dict[str, Callable] = ( + metadata_templates: dict[str, TemplateMetadata] = ( pydough.active_session.metadata.templates_definitions ) From 371caeeb98b56117791987251ba50a0d7019e002 Mon Sep 17 00:00:00 2001 From: john-sanchez31 Date: Mon, 24 Aug 2026 11:26:44 -0600 Subject: [PATCH 03/14] adding internal documentation, fixing call_template api through test functions, also added file for template function tests --- pydough/__init__.py | 3 +- pydough/metadata/parse.py | 14 +- .../metadata/templates/template_metadata.py | 141 +++++++++++++----- pydough/unqualified/__init__.py | 2 + pydough/unqualified/unqualified_node.py | 2 +- pydough/unqualified/unqualified_transform.py | 103 ++++++++++++- tests/test_pipeline_tpch_custom.py | 113 ++++++++++++++ .../test_pydough_functions/tpch_templates.py | 20 +++ tests/testing_utilities.py | 18 ++- 9 files changed, 371 insertions(+), 45 deletions(-) create mode 100644 tests/test_pydough_functions/tpch_templates.py diff --git a/pydough/__init__.py b/pydough/__init__.py index 21b9a7c94..172b5e4ae 100644 --- a/pydough/__init__.py +++ b/pydough/__init__.py @@ -4,6 +4,7 @@ __all__ = [ "active_session", + "call_template", "dataframe_collection", "display_raw", "explain", @@ -26,7 +27,7 @@ from .exploration import explain, explain_llm, explain_structure, explain_term from .logger import get_logger from .metadata import parse_json_metadata_from_file, parse_metadata_from_list -from .unqualified import display_raw, from_string, init_pydough_context +from .unqualified import call_template, display_raw, from_string, init_pydough_context from .user_collections.user_collection_apis import ( dataframe_collection, range_collection, diff --git a/pydough/metadata/parse.py b/pydough/metadata/parse.py index 68bc6de00..3544a7d73 100644 --- a/pydough/metadata/parse.py +++ b/pydough/metadata/parse.py @@ -565,8 +565,18 @@ def parse_template_attributes_v2(graph: GraphMetadata, attribute_json: dict) -> def parse_template_definition_v2(graph: GraphMetadata, definition_json: dict) -> None: """ - TODO - Loads the template definition and save it in the graph property + Parses the JSON object for a PyDough template definition in version 2 of the + PyDough metadata format. + + Args: + `graph`: the metadata for the graph that the template_definition would be + added to. The attribute will be added to this graph in-place. + `attribute_json`: the JSON object containing the metadata for the + template definition. + + Raises: + `PyDoughMetadataException`: if the JSON does not meet the necessary + structure properties. """ template_name: str = extract_string( definition_json, diff --git a/pydough/metadata/templates/template_metadata.py b/pydough/metadata/templates/template_metadata.py index f3b1b25b0..1049c5553 100644 --- a/pydough/metadata/templates/template_metadata.py +++ b/pydough/metadata/templates/template_metadata.py @@ -18,7 +18,7 @@ @dataclass class TemplateParameter: """ - TODO + Dataclass contaning everything needed for the template parameter """ name: str @@ -28,8 +28,9 @@ class TemplateParameter: type: str """ - Type of data this parameter receives. - NOTE: `pydough` type will keep this parameter without type hint. + String containing type of data this parameter receives. + NOTE: `pydough` type will keep this parameter without type hint for the + template. """ description: str @@ -52,6 +53,8 @@ def __init__( code: str, answer_variable: str = "result", ): + super().__init__(description, None, None) + self._graph: GraphMetadata = graph self._name: str = name self._code: str = code @@ -60,49 +63,47 @@ def __init__( self._parameters: dict[str, TemplateParameter] = ( TemplateMetadata.parse_parameters_from_json(parameters) ) - self._template_callable: Callable = self.build_template_callable(graph) - - super().__init__(description, None, None) + self._template_callable: Callable = self.create_template_callable(graph) @property def graph(self) -> GraphMetadata: """ - TODO + The graph that the template belongs to. """ return self._graph @property def name(self) -> str: """ - TODO + Name of the template. """ return self._name @property def parameters(self) -> dict[str, TemplateParameter]: """ - TODO + Parameters required for this template. """ return self._parameters @property def code(self) -> str: """ - TODO + PyDough code being return when the template is called. """ return self._code @property def answer_variable(self) -> str: """ - TODO + Name of the variable being returned and holds the final answer. """ return self._answer_variable @property - def template_callable(self) -> Callable | None: + def template_callable(self) -> Callable: """ - TODO + Template callable used for its execution. """ return self._template_callable @@ -120,15 +121,6 @@ def path(self) -> str: # TODO: Not sure about the usage of this path return f"{self.graph.path}.templates.definitions.{self.name}" - def __call__(self, *args, **kwds): - if self.template_callable is None: - raise ValueError("") - return ( - self.template_callable(*args, **kwds) - if self.template_callable is not None - else None - ) - @staticmethod def create_error_name(name: str, graph_error_name: str): return f"template definition {name!r} in {graph_error_name}" @@ -136,7 +128,20 @@ def create_error_name(name: str, graph_error_name: str): @staticmethod def parse_from_json(graph: GraphMetadata, name: str, definition_json: dict) -> None: """ - TODO + Parses a JSON object into the metadata for a template definition + and inserts it into the graph. + + Args: + `graph`: the metadata for the graph that the template attribute will + be added to. + `name`: the name of the template definition that will be + added to the graph. + `definition_json`: the JSON object that is being parsed to create + the new template definition. + + Raises: + `PyDoughMetadataException`: if the JSON does not meet the necessary + structure properties. """ description: str = extract_string( definition_json, "description", graph.error_name @@ -165,7 +170,19 @@ def parse_parameters_from_json( parameters_json: dict, ) -> dict[str, TemplateParameter]: """ - TODO + Parses a JSON object into the parameters for a template definition + and returns it + + Args: + `parameters_json`: the JSON object that is being parsed to create + the parameters required for a template. + + Returns: + All paremeters parsed for a template. + + Raises: + `PyDoughMetadataException`: if the JSON does not meet the necessary + structure properties. """ template_params: dict[str, TemplateParameter] = {} @@ -188,9 +205,9 @@ def parse_parameters_from_json( return template_params - def build_template_callable( + def create_template_callable( self, - metadata: GraphMetadata | None = None, + graph: GraphMetadata | None = None, ) -> Callable: """ Builds a callable from a PyDough source string, without executing it. @@ -198,21 +215,14 @@ def build_template_callable( `graph.add_template_definition`) and invoked later. Args: - `template_str`: the PyDough code that forms the body of the function. - `args`: the parameter names of the generated function. - `template_name`: the name to give the generated function. - `answer_variable`: the variable in `source` holding the return value. - Defaults to "result". - `metadata`: the metadata graph bound into the function's closure as - `_graph`. Defaults to `pydough.active_session.metadata`. - `environment`: extra names available both when transforming the - source and inside the function's closure. + `graph`: the metadata graph bound into the function's closure as + `_graph`. Returns: The callable built from `source`, not yet invoked. """ - template_str: str = self.generate_template_str() + template_str: str = self.create_template_def() import pydough from pydough.unqualified.unqualified_transform import AddRootVisitor @@ -232,7 +242,7 @@ def build_template_callable( # `_graph` and `pydough` are baked into the function's globals so that # they're available whenever the function is later called namespace: dict[str, Any] = { - graph_name: metadata, + graph_name: graph, "pydough": pydough, } exec(compiled, namespace, namespace) @@ -240,9 +250,21 @@ def build_template_callable( loaded_template: Callable = namespace[self.name] return loaded_template - def generate_template_str(self) -> str: + def create_template_def(self) -> str: """ - TODO + Builds the Python source code for a function definition from this + template's metadata, without executing or compiling it. + + Uses `self.parameters` to construct the function's signature (typed + parameters, except those of type "pydough" which are left untyped), + substitutes positional placeholders (`{1}`, `{2}`, ...) in `self.code` + with the corresponding parameter names, indents the resulting body, + and appends a `return` statement for `self.answer_variable`. + + Returns: + A string containing the full `def (...): ...` source for + this template, ready to be parsed/executed (e.g. via `from_string` + or `exec`) elsewhere to obtain the actual callable. """ # Parsing arguments arg_names: list[str] = list(self.parameters.keys()) @@ -272,3 +294,44 @@ def _replace_placeholder(match: re.Match) -> str: template_str: str = f"def {self.name}({template_args}):\n{indented_code}\n return {self.answer_variable}\n" return template_str + + def create_template_call(self, kwargs: dict[str, dict[str, str | int]] = {}) -> str: + """ + Builds the Python source code for a call to this template, without + executing it. + + Each entry in `kwargs` maps a parameter name to a spec dict with a + `"value"` (rendered as-is into the call) and, optionally, a `"type"` + used to decide formatting: values with type `"str"` are wrapped in + quotes, all others are inserted unquoted (e.g. numeric literals or + pydough expressions such as `customer.market_segment`). + + Args: + `kwargs`: a mapping of parameter name to a spec dict, e.g. + `{"arg_year": {"type": "int", "value": "1996"}}`. Values with + `"type": "str"` are quoted in the generated call; all other + values are inserted as raw, unquoted source text. + + Returns: + A string of the form `"(=, ...)"` representing + the unevaluated call to this template. The returned string is + intended to be embedded in a larger source snippet (e.g. via + `from_string`) rather than executed on its own. + """ + + # TODO: VALIDATE ARG TYPES WITH THE PARAMETERS TYPES + # NOTE: Add test with date types + + args_parts: list = [] + part: str = "" + for key, spec in kwargs.items(): + display_quotes: bool = "type" in spec and spec["type"] == "str" + + arg_value = f"'{spec['value']}'" if display_quotes else f"{spec['value']}" + part = f"{key} = {arg_value}" + + args_parts.append(part) + + args = ", ".join(args_parts) + + return f"""{self.name}({args})""" diff --git a/pydough/unqualified/__init__.py b/pydough/unqualified/__init__.py index 390a03fec..6babd3f06 100644 --- a/pydough/unqualified/__init__.py +++ b/pydough/unqualified/__init__.py @@ -21,6 +21,7 @@ "UnqualifiedTopK", "UnqualifiedWhere", "UnqualifiedWindow", + "call_template", "display_raw", "from_string", "init_pydough_context", @@ -50,6 +51,7 @@ display_raw, ) from .unqualified_transform import ( + call_template, from_string, init_pydough_context, transform_cell, diff --git a/pydough/unqualified/unqualified_node.py b/pydough/unqualified/unqualified_node.py index a73e43891..9b1672421 100644 --- a/pydough/unqualified/unqualified_node.py +++ b/pydough/unqualified/unqualified_node.py @@ -143,7 +143,7 @@ def __call__(self, *args, **kwargs): name = str(self) if name in metadata_templates: - return metadata_templates[name](*args, **kwargs) + return metadata_templates[name].template_callable(*args, **kwargs) raise pydough.active_session.error_builder.undefined_function_call( self, *args, **kwargs diff --git a/pydough/unqualified/unqualified_transform.py b/pydough/unqualified/unqualified_transform.py index 5f0be2885..67b09a0e1 100644 --- a/pydough/unqualified/unqualified_transform.py +++ b/pydough/unqualified/unqualified_transform.py @@ -3,7 +3,13 @@ variables with unqualified nodes by prepending it with `_ROOT.`. """ -__all__ = ["from_string", "init_pydough_context", "transform_cell", "transform_code"] +__all__ = [ + "call_template", + "from_string", + "init_pydough_context", + "transform_cell", + "transform_code", +] import ast import builtins @@ -14,6 +20,7 @@ from pydough.configs import PyDoughSession from pydough.errors import PyDoughSessionException, PyDoughUnqualifiedException from pydough.metadata import GraphMetadata +from pydough.metadata.templates import AttributeMetadata, TemplateMetadata from .unqualified_node import UnqualifiedNode @@ -542,6 +549,100 @@ def from_string( return ret_val +def call_template( + name: str, + labels: dict[str, str], +) -> UnqualifiedNode: + """ + Invokes a named PyDough template using user-facing labels instead of + raw PyDough syntax, returning the resulting unqualified (unexecuted) + node. + + Each entry in `labels` maps a template parameter name to a label string + that is looked up in the active session's graph attributes (via each + attribute's `options`) to resolve it to a concrete value and type. The + resolved arguments are used to build a call to the template as PyDough + source text, which is then parsed and executed via `from_string` to + produce the corresponding `UnqualifiedNode` — the same lazy, chainable + object that would result from calling the template directly with + equivalent literal or expression arguments (e.g. `.WHERE(...)` can be + chained onto the returned node). No query execution occurs at this + point. + + Args: + `name`: the name of the template to call, as registered in + `pydough.active_session.metadata.templates_definitions`. + `labels`: a mapping of template parameter name to a label string + (e.g. `{"arg_year": "Year 1998", "arg_dimension": "Customer Market + Segment"}`). Each label must appear in the `options` of exactly + one attribute in the active session's graph, and that attribute's + type must match the corresponding template parameter's type. + + TODO: What to do with repeated labels? Check before add the option? + + Returns: + The `UnqualifiedNode` produced by calling the template with the + resolved arguments, ready for further chaining or execution + (e.g. `.to_df()`). + + Raises: + `ValueError`: if no metadata is loaded in the active session, if a + label's attribute type doesn't match the template parameter's + expected type, or if a label isn't found in any attribute's + options. + """ + + import pydough + + if pydough.active_session.metadata is None: + raise ValueError("No metadata loaded in the current active session") + + calling_template: TemplateMetadata = ( + pydough.active_session.metadata.templates_definitions[name] + ) + + graph_attributes: dict[str, AttributeMetadata] = ( + pydough.active_session.metadata.templates_attributes + ) + + template_kwargs: dict[str, dict[str, str | int]] = {} + + for arg_name, label in labels.items(): + # Use this arg type to check the option value + arg_type: str = calling_template.parameters[arg_name].type + + kwarg: dict[str, str | int] | None = None + + for attr_name, attribute in graph_attributes.items(): + if label in attribute.options: + # Types must match + if attribute.type != arg_type: + raise ValueError( + f"The type of {attr_name} attribute doesn't match the argument {arg_name} type" + ) + + kwarg = {"type": attribute.type, "value": attribute.options[label]} + # No more search for this label + break + + if kwarg is None: + raise ValueError(f"Label {label} not found in any attribute's options") + + template_kwargs[arg_name] = kwarg + + # Build call + template_call: str = ( + f"result = {calling_template.create_template_call(template_kwargs)}" + ) + + result = from_string( + source=template_call, + metadata=pydough.active_session.metadata, + environment={name: calling_template.template_callable}, + ) + return result + + def init_pydough_context(graph: GraphMetadata): """ Decorator that wraps around a PyDough function and transforms its body into diff --git a/tests/test_pipeline_tpch_custom.py b/tests/test_pipeline_tpch_custom.py index 0c0df5027..83f57c333 100644 --- a/tests/test_pipeline_tpch_custom.py +++ b/tests/test_pipeline_tpch_custom.py @@ -195,6 +195,10 @@ year_month_nation_orders, yoy_change_in_num_orders, ) +from tests.test_pydough_functions.tpch_templates import ( + template_api, + template_call, +) from tests.test_pydough_functions.user_collections import ( dataframe_collection_bad_1, dataframe_collection_bad_2, @@ -5246,6 +5250,115 @@ ), id="literal_template", ), + pytest.param( + PyDoughPandasTest( + "result = pydough.call_template(\n" + " 'orders_revenue_by', {'arg_year': 'Year 1998', 'arg_dimension': 'Customer Market Segment'}" + ")\n", + "TPCH", + lambda: pd.DataFrame( + { + "dimension": [ + "AUTOMOBILE", + "BUILDING", + "FURNITURE", + "HOUSEHOLD", + "MACHINERY", + ], + "segment_revenue": [ + 3.839372e09, + 3.943587e09, + 3.897561e09, + 3.877929e09, + 3.875373e09, + ], + } + ), + "api_template_call_simple", + ), + id="api_template_call_simple", + ), + pytest.param( + PyDoughPandasTest( + "selected_customers = customers.WHERE(" + " (account_balance >= pydough.call_template('template_literal', {'base_number': 'Year 1992'}))\n" + ")\n" + "result = TPCH.CALCULATE(n_custs=COUNT(selected_customers))\n", + "TPCH", + lambda: pd.DataFrame( + { + "dimension": [ + "AUTOMOBILE", + "BUILDING", + "FURNITURE", + "HOUSEHOLD", + "MACHINERY", + ], + "segment_revenue": [ + 3.839372e09, + 3.943587e09, + 3.897561e09, + 3.877929e09, + 3.875373e09, + ], + } + ), + "api_template_call_literal", + ), + id="api_template_call_literal", + ), + pytest.param( + PyDoughPandasTest( + template_call, + "TPCH", + lambda: pd.DataFrame( + { + "dimension": [ + "AUTOMOBILE", + "BUILDING", + "FURNITURE", + "HOUSEHOLD", + "MACHINERY", + ], + "segment_revenue": [ + 6563815859.6161, + 6752822196.8331995, + 6671055612.9225, + 6670709849.0533, + 6620124465.2663, + ], + } + ), + "template_call_func", + ), + id="template_call_func", + ), + pytest.param( + PyDoughPandasTest( + template_api, + "TPCH", + lambda: pd.DataFrame( + { + "dimension": [ + "AFRICA", + "AMERICA", + "ASIA", + "EUROPE", + "MIDDLE EAST", + ], + "segment_revenue": [ + 6.612832e09, + 6.618720e09, + 6.688478e09, + 6.746079e09, + 6.612420e09, + ], + } + ), + "template_api_call", + ), + id="template_api_call", + ), ], ) def tpch_custom_pipeline_test_data(request) -> PyDoughPandasTest: diff --git a/tests/test_pydough_functions/tpch_templates.py b/tests/test_pydough_functions/tpch_templates.py new file mode 100644 index 000000000..27fb879bc --- /dev/null +++ b/tests/test_pydough_functions/tpch_templates.py @@ -0,0 +1,20 @@ +""" +Various functions containing user generated collections as +PyDough code snippets for testing purposes. +""" +# ruff: noqa +# mypy: ignore-errors +# ruff & mypy should not try to typecheck or verify any of this + +import pydough + + +def template_call(): + return orders_revenue_by(1996, customer.market_segment) + + +def template_api(): + return pydough.call_template( + "orders_revenue_by", + labels={"arg_year": "Year 1996", "arg_dimension": "Customer Region"}, + ) diff --git a/tests/testing_utilities.py b/tests/testing_utilities.py index f87e76a22..d1cc42515 100644 --- a/tests/testing_utilities.py +++ b/tests/testing_utilities.py @@ -1053,7 +1053,23 @@ def transform_and_exec_pydough( return pydough.from_string(pydough_impl, metadata=graph, environment=kwargs) else: # Otherwise, transform the function with the decorator and call it. - return init_pydough_context(graph)(pydough_impl)(**kwargs) + + # Temporarily set the active session (or just metadata) so that + # UnqualifiedNode.__call__ can resolve templates via + # pydough.active_session.metadata, and so functions like + # pydough.to_table can access session info during execution. + old_session = pydough.active_session + try: + if session is not None: + pydough.active_session = session + else: + pydough.active_session.metadata = graph + return init_pydough_context(graph)(pydough_impl)(**kwargs) + finally: + if session is not None: + pydough.active_session = old_session + else: + pydough.active_session.metadata = old_session.metadata @dataclass From 6be3926b40e84dbfc6d7740f70bcb440d6225b7f Mon Sep 17 00:00:00 2001 From: john-sanchez31 Date: Mon, 24 Aug 2026 16:03:46 -0600 Subject: [PATCH 04/14] fixing documentation --- pydough/metadata/graphs/graph_metadata.py | 2 -- pydough/metadata/parse.py | 1 - .../metadata/templates/attribute_metadata.py | 11 --------- .../metadata/templates/template_metadata.py | 24 +++++++++---------- tests/test_metadata/sample_graphs.json | 6 ++--- 5 files changed, 14 insertions(+), 30 deletions(-) diff --git a/pydough/metadata/graphs/graph_metadata.py b/pydough/metadata/graphs/graph_metadata.py index afff19001..025c88ac3 100644 --- a/pydough/metadata/graphs/graph_metadata.py +++ b/pydough/metadata/graphs/graph_metadata.py @@ -231,7 +231,6 @@ def add_template_attribute(self, new_attribute: AbstractMetadata) -> None: `PyDoughMetadataException`: if `new_attribute` cannot be inserted into the graph because. """ - from pydough.metadata.templates import AttributeMetadata # Make sure the new_attribute is actually a template_attribute HasType(AttributeMetadata).verify(new_attribute, "attribute") @@ -264,7 +263,6 @@ def add_template_definition( `PyDoughMetadataException`: if `template` cannot be inserted into the graph because. """ - from pydough.metadata.templates import TemplateMetadata # Cirular import error raises if the import is made globally from pydough.pydough_operators import builtin_registered_operators diff --git a/pydough/metadata/parse.py b/pydough/metadata/parse.py index 3544a7d73..359a6a486 100644 --- a/pydough/metadata/parse.py +++ b/pydough/metadata/parse.py @@ -277,7 +277,6 @@ def parse_graph_v2(graph_name: str, graph_json: dict) -> GraphMetadata: for collection in graph.collections.values(): assert isinstance(collection, CollectionMetadata) collection.verify_complete() - return graph diff --git a/pydough/metadata/templates/attribute_metadata.py b/pydough/metadata/templates/attribute_metadata.py index ad01f94c7..85fd61ffd 100644 --- a/pydough/metadata/templates/attribute_metadata.py +++ b/pydough/metadata/templates/attribute_metadata.py @@ -20,17 +20,6 @@ class AttributeMetadata(AbstractMetadata): definitions. """ - # Set of names of fields that can be included in the JSON - # object describing a template attribute. - # TODO: Maybe create a verify complete function with this if not delete it - allowed_fields: set[str] = { - "name", - "usage", - "type", - "description", - "options", - } - def __init__( self, name: str, diff --git a/pydough/metadata/templates/template_metadata.py b/pydough/metadata/templates/template_metadata.py index 1049c5553..234846614 100644 --- a/pydough/metadata/templates/template_metadata.py +++ b/pydough/metadata/templates/template_metadata.py @@ -18,7 +18,7 @@ @dataclass class TemplateParameter: """ - Dataclass contaning everything needed for the template parameter + Parameter for the defined template """ name: str @@ -50,14 +50,14 @@ def __init__( graph: GraphMetadata, description: str, parameters: dict[str, dict[str, str]], - code: str, + source: str, answer_variable: str = "result", ): super().__init__(description, None, None) self._graph: GraphMetadata = graph self._name: str = name - self._code: str = code + self._source: str = source self._answer_variable: str = answer_variable self._parameters: dict[str, TemplateParameter] = ( @@ -87,11 +87,11 @@ def parameters(self) -> dict[str, TemplateParameter]: return self._parameters @property - def code(self) -> str: + def source(self) -> str: """ - PyDough code being return when the template is called. + PyDough source being return when the template is called. """ - return self._code + return self._source @property def answer_variable(self) -> str: @@ -113,7 +113,7 @@ def error_name(self): @property def components(self): - comp: list = [self.name, self.description, self.code] + comp: list = [self.name, self.description, self.source] return comp @property @@ -149,7 +149,7 @@ def parse_from_json(graph: GraphMetadata, name: str, definition_json: dict) -> N answer_variable: str = extract_string( definition_json, "answer_variable", graph.error_name ) - code: str = extract_string(definition_json, "code", graph.error_name) + source: str = extract_string(definition_json, "source", graph.error_name) kwargs: dict[str, dict] = extract_object( definition_json, "parameters", graph.error_name ) @@ -159,7 +159,7 @@ def parse_from_json(graph: GraphMetadata, name: str, definition_json: dict) -> N graph, description, kwargs, - code, + source, answer_variable, ) @@ -277,6 +277,7 @@ def create_template_def(self) -> str: template_args: str = ", ".join(args_parts) # --- Replace {1}, {2}, ... with the corresponding argument name --- + # NOTE: Add a detail documentation about this replacement def _replace_placeholder(match: re.Match) -> str: index = int(match.group(1)) if not (1 <= index <= len(arg_names)): @@ -286,7 +287,7 @@ def _replace_placeholder(match: re.Match) -> str: ) return arg_names[index - 1] - substituted_code = re.sub(r"\{(\d+)\}", _replace_placeholder, self.code) + substituted_code = re.sub(r"\{(\d+)\}", _replace_placeholder, self.source) # --- Indent and assemble the final template --- indented_code: str = textwrap.indent(substituted_code.strip(), " ") @@ -319,9 +320,6 @@ def create_template_call(self, kwargs: dict[str, dict[str, str | int]] = {}) -> `from_string`) rather than executed on its own. """ - # TODO: VALIDATE ARG TYPES WITH THE PARAMETERS TYPES - # NOTE: Add test with date types - args_parts: list = [] part: str = "" for key, spec in kwargs.items(): diff --git a/tests/test_metadata/sample_graphs.json b/tests/test_metadata/sample_graphs.json index 15bd041ba..71b4d3384 100644 --- a/tests/test_metadata/sample_graphs.json +++ b/tests/test_metadata/sample_graphs.json @@ -883,7 +883,7 @@ "dependencies": [], "description": "Calculates the revenue of an order, which is the sum of the extended price * (1 - discount) for all line items in the order.", "parameters": {}, - "code": "result = SUM(lines.extended_price * (1 - lines.discount))\n", + "source": "result = SUM(lines.extended_price * (1 - lines.discount))\n", "answer_variable": "result" }, { @@ -900,7 +900,7 @@ "description": "The dimension by which to partition the revenue calculation." } }, - "code": "result = orders.WHERE(({1} == YEAR(order_date))).CALCULATE(revenue=order_revenue(), dimension=({2})).PARTITION(name=\"orders_groups\", by=dimension).CALCULATE(dimension, segment_revenue=SUM(orders.revenue))\n", + "source": "result = orders.WHERE(({1} == YEAR(order_date))).CALCULATE(revenue=order_revenue(), dimension=({2})).PARTITION(name=\"orders_groups\", by=dimension).CALCULATE(dimension, segment_revenue=SUM(orders.revenue))\n", "answer_variable": "result" }, { @@ -917,7 +917,7 @@ "description": "The metric by which to compare the groups." } }, - "code": "result = {1}.CALCULATE(dimension, segment_revenue, comparison_value={2}).WHERE(ABSENT(PREV(segment_revenue, by=segment_revenue.DESC())) | ABSENT(NEXT(segment_revenue, by=segment_revenue.DESC())) )", + "source": "result = {1}.CALCULATE(dimension, segment_revenue, comparison_value={2}).WHERE(ABSENT(PREV(segment_revenue, by=segment_revenue.DESC())) | ABSENT(NEXT(segment_revenue, by=segment_revenue.DESC())) )", "answer_variable": "result" } ] From 1146278bcf57e5c88f994b7ad2ce5f9f0fde182b Mon Sep 17 00:00:00 2001 From: john-sanchez31 Date: Fri, 4 Sep 2026 08:16:33 -0600 Subject: [PATCH 05/14] adding template test --- demos/metadata/tpch_demo_graph.json | 306 ++++++++++++ pydough/metadata/graphs/graph_metadata.py | 12 + pydough/metadata/parse.py | 4 +- .../metadata/templates/attribute_metadata.py | 19 +- .../metadata/templates/template_metadata.py | 8 +- pydough/unqualified/unqualified_transform.py | 90 +++- tests/conftest.py | 8 + .../databricks_sample_graphs.json | 306 ++++++++++++ tests/test_metadata/invalid_templates.json | 42 ++ tests/test_metadata/sample_graphs.json | 239 +++++++++- .../snowflake_sample_graphs.json | 306 ++++++++++++ tests/test_metadata/trino_graphs.json | 306 ++++++++++++ tests/test_pipeline_tpch_custom.py | 151 ------ .../test_pydough_functions/tpch_templates.py | 114 ++++- tests/test_templates.py | 435 ++++++++++++++++++ 15 files changed, 2162 insertions(+), 184 deletions(-) create mode 100644 tests/test_metadata/invalid_templates.json create mode 100644 tests/test_templates.py diff --git a/demos/metadata/tpch_demo_graph.json b/demos/metadata/tpch_demo_graph.json index 66ee8c7fb..3cd55e6fb 100644 --- a/demos/metadata/tpch_demo_graph.json +++ b/demos/metadata/tpch_demo_graph.json @@ -814,6 +814,312 @@ "synonyms": ["transactions", "purchases"] } ], + "templates": { + "attributes": [ + { + "name": "years", + "usage": {"orders_revenue_by": ["arg_year"], "multiply_by_2": ["base_number"]}, + "type": "int", + "description": "The year for which to calculate the revenue of orders.", + "options": [ + { "label": "Year 1992", "value": 1992}, + { "label": "Year 1993", "value": 1993}, + { "label": "Year 1994", "value": 1994}, + { "label": "Year 1995", "value": 1995}, + { "label": "Year 1996", "value": 1996}, + { "label": "Year 1997", "value": 1997}, + { "label": "Year 1998", "value": 1998} + ] + }, + { + "name": "order_dimensions", + "usage": {"orders_revenue_by": ["arg_dimension"]}, + "type": "pydough", + "description": "The dimension by which to partition the orders calculation.", + "options": [ + {"value": "customer.market_segment", "label": "Customer Market Segment"}, + {"value": "customer.nation.name", "label": "Customer Nation"}, + {"value": "customer.nation.region.name", "label": "Customer Region"}, + {"value": "order_priority", "label": "Order Priority"}, + {"value": "MONTHNAME(order_date)", "label": "Month"}, + {"value": "clerk", "label": "Clerk"} + ] + }, + { + "name": "order_filter_condition", + "usage": {"orders_filter_count": ["orders_filter"]}, + "type": "pydough", + "description": "Condition(s) by which orders can be filtered", + "options": [ + {"label": "Status O", "value": "(order_status == 'O')"}, + {"label": "Price above 3000", "value": "(total_price > 3000)"}, + {"label": "March", "value": "(MONTH(order_date) == 3)"}, + {"label": "High priority", "value": "ISIN(order_priority, ('1-URGENT', '2-HIGH'))"} + ] + }, + { + "name": "order_priority_levels", + "usage": {"order_lvl_priority": ["level"]}, + "type": "str", + "description": "Level of priority for an order", + "options": [ + {"label": "LEVEL 0", "value": "other"}, + {"label": "LEVEL 1", "value": "low"}, + {"label": "LEVEL 2", "value": "medium"}, + {"label": "LEVEL 3", "value": "high"} + ] + }, + { + "name": "dataframe_collection_column", + "usage": {"generate_df_collection": ["col1"]}, + "type": "list", + "description": "List with various types of colors to be used in the dataframe collection", + "options":[ + {"label": "COLORS", "value": "['blue', 'red', 'yellow', 'purple']"} + ] + }, + { + "name": "dataframe_collections_names", + "usage": {"generate_df_collection": ["collection_name"]}, + "type": "str", + "description": "Names of the dataframes to be created in the collection", + "options":[ + {"label": "NAME1", "value": "colors_collection"} + ] + } + ], + "definitions": [ + { + "name": "orders_filter_count", + "description": "Counts the number of orders placed that satisfied the given condition.", + "parameters": { + "orders_filter": { + "type": "pydough", + "description": "Condition(s) for the order to make it count" + } + }, + "source": "result = COUNT(orders.WHERE({1}))", + "answer_variable": "result" + }, + { + "name": "cumulative_orders_counter", + "description": "Calculates the cumulative counter of all orders placed from a base year through a given last year. Recursively sums the current year's counter and the cumulative counter of all next years.", + "parameters": { + "base_year": { "type": "int", "description": "The year through which to calculate cumulative revenue." }, + "last_year": { "type": "int", "description": "The earliest year to include in the cumulation (recursion base case)." } + }, + "source": "assert base_year <= last_year\nif {1} == {2}:\n result = COUNT(orders.WHERE(YEAR(order_date) == base_year))\nelse:\n result = (\n COUNT(orders.WHERE(YEAR(order_date) == base_year)) + cumulative_orders_counter(base_year + 1, last_year))\n", + "answer_variable": "result" + }, + { + "name": "order_revenue", + "description": "Calculates the revenue of an order, which is the sum of the extended price * (1 - discount) for all line items in the order.", + "parameters": {}, + "source": "result = SUM(lines.extended_price * (1 - lines.discount))\n", + "answer_variable": "result" + }, + { + "name": "orders_revenue_by", + "description": "Calculates the revenue of orders in a given year, partitioned by a specified dimension.", + "parameters": { + "arg_year": { + "type": "int", + "description": "The year for which to calculate the revenue of orders." + }, + "arg_dimension": { + "type": "pydough", + "description": "The dimension by which to partition the revenue calculation." + } + }, + "source": "result = orders.WHERE(({1} == YEAR(order_date))).CALCULATE(revenue=order_revenue(), dimension=({2})).PARTITION(name=\"orders_groups\", by=dimension).CALCULATE(dimension, segment_revenue=SUM(orders.revenue))\n", + "answer_variable": "result" + }, + { + "name": "top_bottom_comparison", + "description": "Compares the top and bottom groups of a partitioned orders", + "parameters": { + "arg_partitioned_orders": { + "type": "pydough", + "description": "The partitioned orders to compare." + }, + "arg_calculation": { + "type": "pydough", + "description": "The metric by which to compare the groups." + } + }, + "source": "result = {1}.CALCULATE(dimension, segment_revenue, comparison_value={2}).WHERE(ABSENT(PREV(segment_revenue, by=segment_revenue.DESC())) | ABSENT(NEXT(segment_revenue, by=segment_revenue.DESC())) )", + "answer_variable": "result" + }, + { + "name": "multiply_by_2", + "description": "Receives an integer and return its multiplication by 2.", + "parameters": { + "base_number": { + "type": "int", + "description": "Number being multiply by 2" + } + }, + "source": "result = {1} * 2\n", + "answer_variable": "result" + }, + { + "name": "order_lvl_priority", + "description": "Returns a list with orders priority based on the given level. High, Medium, Low", + "parameters": { + "level": { + "type": "str", + "description": "Level of the priority" + } + }, + "source": "if {1} == 'high':\n result = ['1-URGENT', '2-HIGH']\nelif {1} == 'medium':\n result = ['3-MEDIUM', '5-LOW']\nelse:\n result = ['4-NOT SPECIFIED']", + "answer_variable": "result" + }, + { + "name": "customer_calculate", + "description": "Returns a dictionary, representing the fields for CALCULATE a customer", + "parameters": { + "cust_name": { + "type": "str", + "description": "Rename for the customer name column" + }, + "cust_nation": { + "type": "str", + "description": "Rename for the customer nation name column" + }, + "balance": { + "type": "str", + "description": "Rename for the customer account balance column" + } + }, + "source": "result = {\n {1}: name,\n {2}: nation.name,\n {3}: account_balance\n}", + "answer_variable": "result" + }, + { + "name": "generate_range_collection", + "description": "Generates a simple range collection and returns it", + "parameters":{ + "start": { + "type": "int", + "description": "Start of the range collection" + }, + "end": { + "type": "int", + "description": "End of the range collection" + } + }, + "source": "result = pydough.range_collection('template_range', 'idx', {1}, {2})", + "answer_variable": "result" + }, + { + "name": "range_cross_collection", + "description": "Generates a range collection and cross it with the given collection", + "parameters":{ + "cross_collection": { + "type": "pydough", + "description": "Pydough collection to cross with the generated range collection" + }, + "range_name" : { + "type": "str", + "description": "Name for the generated range collection" + }, + "start": { + "type": "int", + "description": "Start of the range collection" + }, + "end": { + "type": "int", + "description": "End of the range collection" + }, + "cross_cond":{ + "type": "pydough", + "description": "Condition for the cross operation" + } + }, + "source": "created_range = pydough.range_collection({2}, 'idx', {3}, {4}).CALCULATE(idx)\nresult = created_range.CROSS({1}).WHERE({5})", + "answer_variable": "result" + }, + { + "name": "generate_df_collection", + "description": "Generates a dataframe collection from the given parameters", + "parameters": { + "collection_name": { + "type": "str", + "description": "Name for the generated dataframe collection" + }, + "col1": { + "type": "list", + "description": "List with the data for the column of the new dataframe" + } + }, + "source": "assert len(col1) > 0\nnew_df = pd.DataFrame({\n'names': {2}, 'idx': range(len(col1))})\nresult = pydough.dataframe_collection(name={1}, dataframe=new_df, unique_column_names=['idx'])", + "answer_variable": "result" + }, + { + "name": "dataframe_input_collection", + "description": "Generates a dataframe collection from the given parameters", + "parameters": { + "collection_name": { + "type": "str", + "description": "Name for the generated dataframe collection" + }, + "new_df": { + "type": "pd.DataFrame", + "description": "New dataframe to create the collection with" + }, + "unique_columns": { + "type": "list", + "description": "List of unique column names for the dataframe collection" + } + }, + "source": "result = pydough.dataframe_collection({1}, {2}, {3})", + "answer_variable": "result" + }, + { + "name": "temporary_nations", + "description": "Generates a temporary table with nations filtered by the given region", + "parameters": { + "filtered_region": { + "type": "str", + "description": "Region to filter the nations by" + } + }, + "source": "my_nations = nations.WHERE(region.name == {1})\nnations_tmp = pydough.to_table(my_nations, name='region_nations_t1', temp=False, replace=True)\nresult = nations_tmp.CALCULATE(name)", + "answer_variable": "result" + }, + { + "name": "add_datetime_days", + "description": "Adds a specified number of days to a base datetime and returns the resulting datetime", + "parameters": { + "base_datetime": { + "type": "datetime", + "description": "Base datetime to add days to" + }, + "adding_days": { + "type": "int", + "description": "Number of days to add" + } + }, + "source": "result = DATETIME({1}, f'+{{2}} days')", + "answer_variable": "result" + }, + { + "name": "add_datetime_months", + "description": "Adds a specified number of months to a base datetime and returns the resulting datetime", + "parameters": { + "base_datetime": { + "type": "datetime", + "description": "Base datetime to add months to" + }, + "adding_months": { + "type": "int", + "description": "Number of months to add" + } + }, + "source": "result = pd.to_datetime({1}) + pd.DateOffset(months={2})", + "answer_variable": "result" + } + ] + }, "additional definitions": [], "verified pydough analysis": [], "extra semantic info": {} diff --git a/pydough/metadata/graphs/graph_metadata.py b/pydough/metadata/graphs/graph_metadata.py index 025c88ac3..c53b12593 100644 --- a/pydough/metadata/graphs/graph_metadata.py +++ b/pydough/metadata/graphs/graph_metadata.py @@ -220,6 +220,16 @@ def add_function(self, name: str, function: "ExpressionFunctionOperator") -> Non ) self.functions[name] = function + def get_all_labels(self) -> dict[str, str]: + """ + Fetches all of the labels defined in the graph's template attributes. + """ + all_labels: dict[str, str] = {} + for attribute in self.templates_attributes.values(): + all_labels.update(dict.fromkeys(attribute.options.keys(), attribute.name)) + + return all_labels + def add_template_attribute(self, new_attribute: AbstractMetadata) -> None: """ Adds a new attribute to the graph. @@ -231,6 +241,7 @@ def add_template_attribute(self, new_attribute: AbstractMetadata) -> None: `PyDoughMetadataException`: if `new_attribute` cannot be inserted into the graph because. """ + from pydough.metadata.templates import AttributeMetadata # Make sure the new_attribute is actually a template_attribute HasType(AttributeMetadata).verify(new_attribute, "attribute") @@ -265,6 +276,7 @@ def add_template_definition( """ # Cirular import error raises if the import is made globally + from pydough.metadata.templates import TemplateMetadata from pydough.pydough_operators import builtin_registered_operators # Make sure the new_template is actually a template_attribute diff --git a/pydough/metadata/parse.py b/pydough/metadata/parse.py index 359a6a486..79c7bee5d 100644 --- a/pydough/metadata/parse.py +++ b/pydough/metadata/parse.py @@ -255,7 +255,7 @@ def parse_graph_v2(graph_name: str, graph_json: dict) -> GraphMetadata: for attribute_definition in attribute_definitions: is_json_object.verify( attribute_definition, - f"metadat for Templates definition inside {graph.error_name}", + f"metadata for Templates definition inside {graph.error_name}", ) assert isinstance(attribute_definition, dict) parse_template_attributes_v2(graph, attribute_definition) @@ -268,7 +268,7 @@ def parse_graph_v2(graph_name: str, graph_json: dict) -> GraphMetadata: for template_definition in templates_definitions: is_json_object.verify( template_definition, - f"metadat for Templates definition inside {graph.error_name}", + f"metadata for Templates definition inside {graph.error_name}", ) assert isinstance(template_definition, dict) parse_template_definition_v2(graph, template_definition) diff --git a/pydough/metadata/templates/attribute_metadata.py b/pydough/metadata/templates/attribute_metadata.py index 85fd61ffd..81a0dcfcd 100644 --- a/pydough/metadata/templates/attribute_metadata.py +++ b/pydough/metadata/templates/attribute_metadata.py @@ -7,6 +7,7 @@ HasType, extract_array, extract_integer, + extract_object, extract_string, ) from pydough.metadata.abstract_metadata import AbstractMetadata @@ -24,7 +25,7 @@ def __init__( self, name: str, graph: GraphMetadata, - usage: list[str], + usage: dict[str, list[str]], type: str, description: str, ): @@ -32,7 +33,7 @@ def __init__( self._graph: GraphMetadata = graph self._name: str = name - self._usage: list[str] = usage + self._usage: dict[str, list[str]] = usage self._type: str = type self._options: dict[str, str | int] = {} @@ -53,7 +54,7 @@ def name(self) -> str: return self._name @property - def usage(self) -> list[str]: + def usage(self) -> dict[str, list[str]]: """ List with the names of the templates where the attribute can be used """ @@ -131,7 +132,9 @@ def parse_from_json( # Extract the relevant properties from the JSON to build the new template # attribute, then add it to the graph - attr_usage: list[str] = extract_array(attribute_json, "usage", error_name) + attr_usage: dict[str, list[str]] = extract_object( + attribute_json, "usage", error_name + ) attr_type: str = extract_string(attribute_json, "type", error_name) attr_desc: str = extract_string(attribute_json, "description", error_name) @@ -158,6 +161,14 @@ def parse_from_json( except PyDoughMetadataException: raise PyDoughMetadataException("Option value must be string or integer") + graph_labels: dict[str, str] = graph.get_all_labels() + if label in graph_labels: + raise ValueError( + f"Duplicate option label: {label!r} for attribute {attribute_name!r}. " + f"The label is already in use by attribute {graph_labels[label]!r} " + f"in graph {graph.name!r}." + ) + new_attribute.add_attribute_option(label, value) graph.add_template_attribute(new_attribute) diff --git a/pydough/metadata/templates/template_metadata.py b/pydough/metadata/templates/template_metadata.py index 234846614..e6a0680c4 100644 --- a/pydough/metadata/templates/template_metadata.py +++ b/pydough/metadata/templates/template_metadata.py @@ -229,7 +229,7 @@ def create_template_callable( # Args + "pydough" are "known" so they aren't # rewritten into _ROOT. by the visitor. - known_names: set[str] = {"pydough"} + known_names: set[str] = {"pydough", "pd"} graph_name: str = "_graph" visitor = AddRootVisitor(graph_name, known_names) @@ -239,11 +239,15 @@ def create_template_callable( compiled = compile(transformed_code, filename=f"<{self.name}>", mode="exec") - # `_graph` and `pydough` are baked into the function's globals so that + # `_graph`, `pydough` and pd are baked into the function's globals so that # they're available whenever the function is later called + # Required for dataframe collections + import pandas as pd + namespace: dict[str, Any] = { graph_name: graph, "pydough": pydough, + "pd": pd, } exec(compiled, namespace, namespace) diff --git a/pydough/unqualified/unqualified_transform.py b/pydough/unqualified/unqualified_transform.py index 67b09a0e1..38f4968e9 100644 --- a/pydough/unqualified/unqualified_transform.py +++ b/pydough/unqualified/unqualified_transform.py @@ -459,6 +459,54 @@ def from_string( A PyDough UnualifiedNode object representing the result of the transformed PyDough code. + Raises: + `PyDoughSessionException` if both `session` and `metadata` are provided. + """ + + source_execution = _execute_source( + source, answer_variable, metadata, environment, session + ) + assert isinstance(source_execution, UnqualifiedNode) + + return source_execution + + +def _execute_source( + source: str, + answer_variable: str | None = None, + metadata: GraphMetadata | None = None, + environment: dict[str, Any] | None = None, + session: PyDoughSession | None = None, + return_unqualified: bool = True, +) -> Any: + """ + Parses and transforms a PyDough source string, returning an unqualified node + if `return_unqualified` is True on which operations like `explain()`, `to_sql()` + , or `to_df()` can be called, otherwise could return a literal. + + Args: + `source`: a valid PyDough code string that will be executed to define + the PyDough code. + `answer_variable`: The name of the variable that holds the result of the + PyDough code. If not provided, assumes the answer is `result`. + `metadata`: The metadata graph to use. If not provided, + `active_session.metadata` will be used. + `environment`: A dictionary of variables that will be available + in the environment where the PyDough code is executed. If not provided, + uses an empty dictionary. + `session`: A PyDoughSession to use during execution. If provided, this + session will be temporarily bound to `pydough.active_session` during + code execution, allowing functions like `pydough.to_table` to access + the database connection. If not provided, only metadata is temporarily + set on the active session. Cannot be combined with `metadata` — if a + session is provided, use `session.metadata` to control the graph. + `return_unqualified`: Flag that triggers the returned type, if True + an UnqualifiedNode will be returned, Any otherwise. + + Returns: + A PyDough UnualifiedNode object representing the result of the + transformed PyDough code. + Raises: `PyDoughSessionException` if both `session` and `metadata` are provided. """ @@ -541,8 +589,8 @@ def from_string( f"PyDough code expected to store the answer in a variable named '{answer_variable}'." ) ret_val = execution_context[answer_variable] - # Check if answer is an UnqualifiedNode - if not isinstance(ret_val, UnqualifiedNode): + # if return_unqualified is True, make sure the answer is an UnqualifiedNode + if return_unqualified and not isinstance(ret_val, UnqualifiedNode): raise PyDoughUnqualifiedException( f"Expected variable {answer_variable!r} in the text to store PyDough code, instead found {ret_val.__class__.__name__!r}." ) @@ -552,7 +600,7 @@ def from_string( def call_template( name: str, labels: dict[str, str], -) -> UnqualifiedNode: +) -> Any: """ Invokes a named PyDough template using user-facing labels instead of raw PyDough syntax, returning the resulting unqualified (unexecuted) @@ -562,12 +610,9 @@ def call_template( that is looked up in the active session's graph attributes (via each attribute's `options`) to resolve it to a concrete value and type. The resolved arguments are used to build a call to the template as PyDough - source text, which is then parsed and executed via `from_string` to - produce the corresponding `UnqualifiedNode` — the same lazy, chainable - object that would result from calling the template directly with - equivalent literal or expression arguments (e.g. `.WHERE(...)` can be - chained onto the returned node). No query execution occurs at this - point. + source text, which is then parsed and executed via `_execute_source` to + produce the corresponding result, which can be an `UnqualifiedNode` + or any other type. Args: `name`: the name of the template to call, as registered in @@ -601,10 +646,15 @@ def call_template( pydough.active_session.metadata.templates_definitions[name] ) - graph_attributes: dict[str, AttributeMetadata] = ( + available_attributes: dict[str, AttributeMetadata] = ( pydough.active_session.metadata.templates_attributes ) + if len(available_attributes) == 0: + raise ValueError( + f"No attributes available for template '{name}' in the current active session" + ) + template_kwargs: dict[str, dict[str, str | int]] = {} for arg_name, label in labels.items(): @@ -613,12 +663,22 @@ def call_template( kwarg: dict[str, str | int] | None = None - for attr_name, attribute in graph_attributes.items(): + for attr_name, attribute in available_attributes.items(): if label in attribute.options: + # Check if the attribute is available for this template and argument + if attribute.usage != {} and ( + name not in attribute.usage + or attribute.usage[name] == [] # No argument restrictions + or arg_name not in attribute.usage[name] + ): + raise ValueError( + f"The attribute '{attr_name}' is not available for parameter '{arg_name}' on template '{name}'" + ) + # Types must match if attribute.type != arg_type: raise ValueError( - f"The type of {attr_name} attribute doesn't match the argument {arg_name} type" + f"The type of '{attr_name}' attribute doesn't match the argument '{arg_name}'s type" ) kwarg = {"type": attribute.type, "value": attribute.options[label]} @@ -634,11 +694,13 @@ def call_template( template_call: str = ( f"result = {calling_template.create_template_call(template_kwargs)}" ) + import pandas as pd - result = from_string( + result = _execute_source( source=template_call, metadata=pydough.active_session.metadata, - environment={name: calling_template.template_callable}, + environment={name: calling_template.template_callable, "pd": pd}, + return_unqualified=False, ) return result diff --git a/tests/conftest.py b/tests/conftest.py index 0502f0ad9..1e2659372 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -249,6 +249,14 @@ def invalid_graph_path() -> str: return f"{os.path.dirname(__file__)}/test_metadata/invalid_graphs.json" +@pytest.fixture(scope="session") +def invalid_templates_graph_path() -> str: + """ + Tuple of the path to the JSON file containing the invalid templates. + """ + return f"{os.path.dirname(__file__)}/test_metadata/invalid_templates.json" + + @pytest.fixture(scope="session") def valid_sample_graph_names() -> set[str]: """ diff --git a/tests/test_metadata/databricks_sample_graphs.json b/tests/test_metadata/databricks_sample_graphs.json index a430cb0af..18b0462f7 100644 --- a/tests/test_metadata/databricks_sample_graphs.json +++ b/tests/test_metadata/databricks_sample_graphs.json @@ -814,6 +814,312 @@ "synonyms": ["transactions", "purchases"] } ], + "templates": { + "attributes": [ + { + "name": "years", + "usage": {"orders_revenue_by": ["arg_year"], "multiply_by_2": ["base_number"]}, + "type": "int", + "description": "The year for which to calculate the revenue of orders.", + "options": [ + { "label": "Year 1992", "value": 1992}, + { "label": "Year 1993", "value": 1993}, + { "label": "Year 1994", "value": 1994}, + { "label": "Year 1995", "value": 1995}, + { "label": "Year 1996", "value": 1996}, + { "label": "Year 1997", "value": 1997}, + { "label": "Year 1998", "value": 1998} + ] + }, + { + "name": "order_dimensions", + "usage": {"orders_revenue_by": ["arg_dimension"]}, + "type": "pydough", + "description": "The dimension by which to partition the orders calculation.", + "options": [ + {"value": "customer.market_segment", "label": "Customer Market Segment"}, + {"value": "customer.nation.name", "label": "Customer Nation"}, + {"value": "customer.nation.region.name", "label": "Customer Region"}, + {"value": "order_priority", "label": "Order Priority"}, + {"value": "MONTHNAME(order_date)", "label": "Month"}, + {"value": "clerk", "label": "Clerk"} + ] + }, + { + "name": "order_filter_condition", + "usage": {"orders_filter_count": ["orders_filter"]}, + "type": "pydough", + "description": "Condition(s) by which orders can be filtered", + "options": [ + {"label": "Status O", "value": "(order_status == 'O')"}, + {"label": "Price above 3000", "value": "(total_price > 3000)"}, + {"label": "March", "value": "(MONTH(order_date) == 3)"}, + {"label": "High priority", "value": "ISIN(order_priority, ('1-URGENT', '2-HIGH'))"} + ] + }, + { + "name": "order_priority_levels", + "usage": {"order_lvl_priority": ["level"]}, + "type": "str", + "description": "Level of priority for an order", + "options": [ + {"label": "LEVEL 0", "value": "other"}, + {"label": "LEVEL 1", "value": "low"}, + {"label": "LEVEL 2", "value": "medium"}, + {"label": "LEVEL 3", "value": "high"} + ] + }, + { + "name": "dataframe_collection_column", + "usage": {"generate_df_collection": ["col1"]}, + "type": "list", + "description": "List with various types of colors to be used in the dataframe collection", + "options":[ + {"label": "COLORS", "value": "['blue', 'red', 'yellow', 'purple']"} + ] + }, + { + "name": "dataframe_collections_names", + "usage": {"generate_df_collection": ["collection_name"]}, + "type": "str", + "description": "Names of the dataframes to be created in the collection", + "options":[ + {"label": "NAME1", "value": "colors_collection"} + ] + } + ], + "definitions": [ + { + "name": "orders_filter_count", + "description": "Counts the number of orders placed that satisfied the given condition.", + "parameters": { + "orders_filter": { + "type": "pydough", + "description": "Condition(s) for the order to make it count" + } + }, + "source": "result = COUNT(orders.WHERE({1}))", + "answer_variable": "result" + }, + { + "name": "cumulative_orders_counter", + "description": "Calculates the cumulative counter of all orders placed from a base year through a given last year. Recursively sums the current year's counter and the cumulative counter of all next years.", + "parameters": { + "base_year": { "type": "int", "description": "The year through which to calculate cumulative revenue." }, + "last_year": { "type": "int", "description": "The earliest year to include in the cumulation (recursion base case)." } + }, + "source": "assert base_year <= last_year\nif {1} == {2}:\n result = COUNT(orders.WHERE(YEAR(order_date) == base_year))\nelse:\n result = (\n COUNT(orders.WHERE(YEAR(order_date) == base_year)) + cumulative_orders_counter(base_year + 1, last_year))\n", + "answer_variable": "result" + }, + { + "name": "order_revenue", + "description": "Calculates the revenue of an order, which is the sum of the extended price * (1 - discount) for all line items in the order.", + "parameters": {}, + "source": "result = SUM(lines.extended_price * (1 - lines.discount))\n", + "answer_variable": "result" + }, + { + "name": "orders_revenue_by", + "description": "Calculates the revenue of orders in a given year, partitioned by a specified dimension.", + "parameters": { + "arg_year": { + "type": "int", + "description": "The year for which to calculate the revenue of orders." + }, + "arg_dimension": { + "type": "pydough", + "description": "The dimension by which to partition the revenue calculation." + } + }, + "source": "result = orders.WHERE(({1} == YEAR(order_date))).CALCULATE(revenue=order_revenue(), dimension=({2})).PARTITION(name=\"orders_groups\", by=dimension).CALCULATE(dimension, segment_revenue=SUM(orders.revenue))\n", + "answer_variable": "result" + }, + { + "name": "top_bottom_comparison", + "description": "Compares the top and bottom groups of a partitioned orders", + "parameters": { + "arg_partitioned_orders": { + "type": "pydough", + "description": "The partitioned orders to compare." + }, + "arg_calculation": { + "type": "pydough", + "description": "The metric by which to compare the groups." + } + }, + "source": "result = {1}.CALCULATE(dimension, segment_revenue, comparison_value={2}).WHERE(ABSENT(PREV(segment_revenue, by=segment_revenue.DESC())) | ABSENT(NEXT(segment_revenue, by=segment_revenue.DESC())) )", + "answer_variable": "result" + }, + { + "name": "multiply_by_2", + "description": "Receives an integer and return its multiplication by 2.", + "parameters": { + "base_number": { + "type": "int", + "description": "Number being multiply by 2" + } + }, + "source": "result = {1} * 2\n", + "answer_variable": "result" + }, + { + "name": "order_lvl_priority", + "description": "Returns a list with orders priority based on the given level. High, Medium, Low", + "parameters": { + "level": { + "type": "str", + "description": "Level of the priority" + } + }, + "source": "if {1} == 'high':\n result = ['1-URGENT', '2-HIGH']\nelif {1} == 'medium':\n result = ['3-MEDIUM', '5-LOW']\nelse:\n result = ['4-NOT SPECIFIED']", + "answer_variable": "result" + }, + { + "name": "customer_calculate", + "description": "Returns a dictionary, representing the fields for CALCULATE a customer", + "parameters": { + "cust_name": { + "type": "str", + "description": "Rename for the customer name column" + }, + "cust_nation": { + "type": "str", + "description": "Rename for the customer nation name column" + }, + "balance": { + "type": "str", + "description": "Rename for the customer account balance column" + } + }, + "source": "result = {\n {1}: name,\n {2}: nation.name,\n {3}: account_balance\n}", + "answer_variable": "result" + }, + { + "name": "generate_range_collection", + "description": "Generates a simple range collection and returns it", + "parameters":{ + "start": { + "type": "int", + "description": "Start of the range collection" + }, + "end": { + "type": "int", + "description": "End of the range collection" + } + }, + "source": "result = pydough.range_collection('template_range', 'idx', {1}, {2})", + "answer_variable": "result" + }, + { + "name": "range_cross_collection", + "description": "Generates a range collection and cross it with the given collection", + "parameters":{ + "cross_collection": { + "type": "pydough", + "description": "Pydough collection to cross with the generated range collection" + }, + "range_name" : { + "type": "str", + "description": "Name for the generated range collection" + }, + "start": { + "type": "int", + "description": "Start of the range collection" + }, + "end": { + "type": "int", + "description": "End of the range collection" + }, + "cross_cond":{ + "type": "pydough", + "description": "Condition for the cross operation" + } + }, + "source": "created_range = pydough.range_collection({2}, 'idx', {3}, {4}).CALCULATE(idx)\nresult = created_range.CROSS({1}).WHERE({5})", + "answer_variable": "result" + }, + { + "name": "generate_df_collection", + "description": "Generates a dataframe collection from the given parameters", + "parameters": { + "collection_name": { + "type": "str", + "description": "Name for the generated dataframe collection" + }, + "col1": { + "type": "list", + "description": "List with the data for the column of the new dataframe" + } + }, + "source": "assert len(col1) > 0\nnew_df = pd.DataFrame({\n'names': {2}, 'idx': range(len(col1))})\nresult = pydough.dataframe_collection(name={1}, dataframe=new_df, unique_column_names=['idx'])", + "answer_variable": "result" + }, + { + "name": "dataframe_input_collection", + "description": "Generates a dataframe collection from the given parameters", + "parameters": { + "collection_name": { + "type": "str", + "description": "Name for the generated dataframe collection" + }, + "new_df": { + "type": "pd.DataFrame", + "description": "New dataframe to create the collection with" + }, + "unique_columns": { + "type": "list", + "description": "List of unique column names for the dataframe collection" + } + }, + "source": "result = pydough.dataframe_collection({1}, {2}, {3})", + "answer_variable": "result" + }, + { + "name": "temporary_nations", + "description": "Generates a temporary table with nations filtered by the given region", + "parameters": { + "filtered_region": { + "type": "str", + "description": "Region to filter the nations by" + } + }, + "source": "my_nations = nations.WHERE(region.name == {1})\nnations_tmp = pydough.to_table(my_nations, name='region_nations_t1', temp=False, replace=True)\nresult = nations_tmp.CALCULATE(name)", + "answer_variable": "result" + }, + { + "name": "add_datetime_days", + "description": "Adds a specified number of days to a base datetime and returns the resulting datetime", + "parameters": { + "base_datetime": { + "type": "datetime", + "description": "Base datetime to add days to" + }, + "adding_days": { + "type": "int", + "description": "Number of days to add" + } + }, + "source": "result = DATETIME({1}, f'+{{2}} days')", + "answer_variable": "result" + }, + { + "name": "add_datetime_months", + "description": "Adds a specified number of months to a base datetime and returns the resulting datetime", + "parameters": { + "base_datetime": { + "type": "datetime", + "description": "Base datetime to add months to" + }, + "adding_months": { + "type": "int", + "description": "Number of months to add" + } + }, + "source": "result = pd.to_datetime({1}) + pd.DateOffset(months={2})", + "answer_variable": "result" + } + ] + }, "additional definitions": [], "verified pydough analysis": [], "extra semantic info": {} diff --git a/tests/test_metadata/invalid_templates.json b/tests/test_metadata/invalid_templates.json new file mode 100644 index 000000000..fa50566b9 --- /dev/null +++ b/tests/test_metadata/invalid_templates.json @@ -0,0 +1,42 @@ +[ + { + "name": "NO_TEMPLATES_DEFINITIONS", + "version": "V2", + "relationships": [], + "collections": [], + "templates":{ + "attributes": [ + { + "name": "attr1", + "usage": {}, + "type": "str", + "description": "Generic desctiption", + "options": [ + { "label": "LABEL 1", "value": "VALUE 1"}, + { "label": "LABEL 2", "value": "VALUE 2"}, + { "label": "LABEL 3", "value": "VALUE 3"} + ] + } + ] + } + }, + { + "name": "NO_ATTIBUTE_OPTIONS", + "version": "V2", + "relationships": [], + "collections": [], + "templates":{ + "attributes": [ + { + "name": "attr1", + "usage": {}, + "type": "str", + "description": "Generic desctiption", + "options": [ + ] + } + ], + "definitions":[] + } + } +] \ No newline at end of file diff --git a/tests/test_metadata/sample_graphs.json b/tests/test_metadata/sample_graphs.json index 71b4d3384..4c990db71 100644 --- a/tests/test_metadata/sample_graphs.json +++ b/tests/test_metadata/sample_graphs.json @@ -849,7 +849,7 @@ "attributes": [ { "name": "years", - "usage": [], + "usage": {"orders_revenue_by": ["arg_year"], "multiply_by_2": ["base_number"]}, "type": "int", "description": "The year for which to calculate the revenue of orders.", "options": [ @@ -864,7 +864,7 @@ }, { "name": "order_dimensions", - "usage": ["orders_revenue_by"], + "usage": {"orders_revenue_by": ["arg_dimension"]}, "type": "pydough", "description": "The dimension by which to partition the orders calculation.", "options": [ @@ -875,12 +875,75 @@ {"value": "MONTHNAME(order_date)", "label": "Month"}, {"value": "clerk", "label": "Clerk"} ] + }, + { + "name": "order_filter_condition", + "usage": {"orders_filter_count": ["orders_filter"]}, + "type": "pydough", + "description": "Condition(s) by which orders can be filtered", + "options": [ + {"label": "Status O", "value": "(order_status == 'O')"}, + {"label": "Price above 3000", "value": "(total_price > 3000)"}, + {"label": "March", "value": "(MONTH(order_date) == 3)"}, + {"label": "High priority", "value": "ISIN(order_priority, ('1-URGENT', '2-HIGH'))"} + ] + }, + { + "name": "order_priority_levels", + "usage": {"order_lvl_priority": ["level"]}, + "type": "str", + "description": "Level of priority for an order", + "options": [ + {"label": "LEVEL 0", "value": "other"}, + {"label": "LEVEL 1", "value": "low"}, + {"label": "LEVEL 2", "value": "medium"}, + {"label": "LEVEL 3", "value": "high"} + ] + }, + { + "name": "dataframe_collection_column", + "usage": {"generate_df_collection": ["col1"]}, + "type": "list", + "description": "List with various types of colors to be used in the dataframe collection", + "options":[ + {"label": "COLORS", "value": "['blue', 'red', 'yellow', 'purple']"} + ] + }, + { + "name": "dataframe_collections_names", + "usage": {"generate_df_collection": ["collection_name"]}, + "type": "str", + "description": "Names of the dataframes to be created in the collection", + "options":[ + {"label": "NAME1", "value": "colors_collection"} + ] } ], "definitions": [ + { + "name": "orders_filter_count", + "description": "Counts the number of orders placed that satisfied the given condition.", + "parameters": { + "orders_filter": { + "type": "pydough", + "description": "Condition(s) for the order to make it count" + } + }, + "source": "result = COUNT(orders.WHERE({1}))", + "answer_variable": "result" + }, + { + "name": "cumulative_orders_counter", + "description": "Calculates the cumulative counter of all orders placed from a base year through a given last year. Recursively sums the current year's counter and the cumulative counter of all next years.", + "parameters": { + "base_year": { "type": "int", "description": "The year through which to calculate cumulative revenue." }, + "last_year": { "type": "int", "description": "The earliest year to include in the cumulation (recursion base case)." } + }, + "source": "assert base_year <= last_year\nif {1} == {2}:\n result = COUNT(orders.WHERE(YEAR(order_date) == base_year))\nelse:\n result = (\n COUNT(orders.WHERE(YEAR(order_date) == base_year)) + cumulative_orders_counter(base_year + 1, last_year))\n", + "answer_variable": "result" + }, { "name": "order_revenue", - "dependencies": [], "description": "Calculates the revenue of an order, which is the sum of the extended price * (1 - discount) for all line items in the order.", "parameters": {}, "source": "result = SUM(lines.extended_price * (1 - lines.discount))\n", @@ -889,7 +952,6 @@ { "name": "orders_revenue_by", "description": "Calculates the revenue of orders in a given year, partitioned by a specified dimension.", - "dependencies": ["order_revenue"], "parameters": { "arg_year": { "type": "int", @@ -906,7 +968,6 @@ { "name": "top_bottom_comparison", "description": "Compares the top and bottom groups of a partitioned orders", - "dependencies": [], "parameters": { "arg_partitioned_orders": { "type": "pydough", @@ -919,6 +980,174 @@ }, "source": "result = {1}.CALCULATE(dimension, segment_revenue, comparison_value={2}).WHERE(ABSENT(PREV(segment_revenue, by=segment_revenue.DESC())) | ABSENT(NEXT(segment_revenue, by=segment_revenue.DESC())) )", "answer_variable": "result" + }, + { + "name": "multiply_by_2", + "description": "Receives an integer and return its multiplication by 2.", + "parameters": { + "base_number": { + "type": "int", + "description": "Number being multiply by 2" + } + }, + "source": "result = {1} * 2\n", + "answer_variable": "result" + }, + { + "name": "order_lvl_priority", + "description": "Returns a list with orders priority based on the given level. High, Medium, Low", + "parameters": { + "level": { + "type": "str", + "description": "Level of the priority" + } + }, + "source": "if {1} == 'high':\n result = ['1-URGENT', '2-HIGH']\nelif {1} == 'medium':\n result = ['3-MEDIUM', '5-LOW']\nelse:\n result = ['4-NOT SPECIFIED']", + "answer_variable": "result" + }, + { + "name": "customer_calculate", + "description": "Returns a dictionary, representing the fields for CALCULATE a customer", + "parameters": { + "cust_name": { + "type": "str", + "description": "Rename for the customer name column" + }, + "cust_nation": { + "type": "str", + "description": "Rename for the customer nation name column" + }, + "balance": { + "type": "str", + "description": "Rename for the customer account balance column" + } + }, + "source": "result = {\n {1}: name,\n {2}: nation.name,\n {3}: account_balance\n}", + "answer_variable": "result" + }, + { + "name": "generate_range_collection", + "description": "Generates a simple range collection and returns it", + "parameters":{ + "start": { + "type": "int", + "description": "Start of the range collection" + }, + "end": { + "type": "int", + "description": "End of the range collection" + } + }, + "source": "result = pydough.range_collection('template_range', 'idx', {1}, {2})", + "answer_variable": "result" + }, + { + "name": "range_cross_collection", + "description": "Generates a range collection and cross it with the given collection", + "parameters":{ + "cross_collection": { + "type": "pydough", + "description": "Pydough collection to cross with the generated range collection" + }, + "range_name" : { + "type": "str", + "description": "Name for the generated range collection" + }, + "start": { + "type": "int", + "description": "Start of the range collection" + }, + "end": { + "type": "int", + "description": "End of the range collection" + }, + "cross_cond":{ + "type": "pydough", + "description": "Condition for the cross operation" + } + }, + "source": "created_range = pydough.range_collection({2}, 'idx', {3}, {4}).CALCULATE(idx)\nresult = created_range.CROSS({1}).WHERE({5})", + "answer_variable": "result" + }, + { + "name": "generate_df_collection", + "description": "Generates a dataframe collection from the given parameters", + "parameters": { + "collection_name": { + "type": "str", + "description": "Name for the generated dataframe collection" + }, + "col1": { + "type": "list", + "description": "List with the data for the column of the new dataframe" + } + }, + "source": "assert len(col1) > 0\nnew_df = pd.DataFrame({\n'names': {2}, 'idx': range(len(col1))})\nresult = pydough.dataframe_collection(name={1}, dataframe=new_df, unique_column_names=['idx'])", + "answer_variable": "result" + }, + { + "name": "dataframe_input_collection", + "description": "Generates a dataframe collection from the given parameters", + "parameters": { + "collection_name": { + "type": "str", + "description": "Name for the generated dataframe collection" + }, + "new_df": { + "type": "pd.DataFrame", + "description": "New dataframe to create the collection with" + }, + "unique_columns": { + "type": "list", + "description": "List of unique column names for the dataframe collection" + } + }, + "source": "result = pydough.dataframe_collection({1}, {2}, {3})", + "answer_variable": "result" + }, + { + "name": "temporary_nations", + "description": "Generates a temporary table with nations filtered by the given region", + "parameters": { + "filtered_region": { + "type": "str", + "description": "Region to filter the nations by" + } + }, + "source": "my_nations = nations.WHERE(region.name == {1})\nnations_tmp = pydough.to_table(my_nations, name='region_nations_t1', temp=False, replace=True)\nresult = nations_tmp.CALCULATE(name)", + "answer_variable": "result" + }, + { + "name": "add_datetime_days", + "description": "Adds a specified number of days to a base datetime and returns the resulting datetime", + "parameters": { + "base_datetime": { + "type": "datetime", + "description": "Base datetime to add days to" + }, + "adding_days": { + "type": "int", + "description": "Number of days to add" + } + }, + "source": "result = DATETIME({1}, f'+{{2}} days')", + "answer_variable": "result" + }, + { + "name": "add_datetime_months", + "description": "Adds a specified number of months to a base datetime and returns the resulting datetime", + "parameters": { + "base_datetime": { + "type": "datetime", + "description": "Base datetime to add months to" + }, + "adding_months": { + "type": "int", + "description": "Number of months to add" + } + }, + "source": "result = pd.to_datetime({1}) + pd.DateOffset(months={2})", + "answer_variable": "result" } ] }, diff --git a/tests/test_metadata/snowflake_sample_graphs.json b/tests/test_metadata/snowflake_sample_graphs.json index 8e69f3a5b..81198c1a4 100644 --- a/tests/test_metadata/snowflake_sample_graphs.json +++ b/tests/test_metadata/snowflake_sample_graphs.json @@ -814,6 +814,312 @@ "synonyms": ["transactions", "purchases"] } ], + "templates": { + "attributes": [ + { + "name": "years", + "usage": {"orders_revenue_by": ["arg_year"], "multiply_by_2": ["base_number"]}, + "type": "int", + "description": "The year for which to calculate the revenue of orders.", + "options": [ + { "label": "Year 1992", "value": 1992}, + { "label": "Year 1993", "value": 1993}, + { "label": "Year 1994", "value": 1994}, + { "label": "Year 1995", "value": 1995}, + { "label": "Year 1996", "value": 1996}, + { "label": "Year 1997", "value": 1997}, + { "label": "Year 1998", "value": 1998} + ] + }, + { + "name": "order_dimensions", + "usage": {"orders_revenue_by": ["arg_dimension"]}, + "type": "pydough", + "description": "The dimension by which to partition the orders calculation.", + "options": [ + {"value": "customer.market_segment", "label": "Customer Market Segment"}, + {"value": "customer.nation.name", "label": "Customer Nation"}, + {"value": "customer.nation.region.name", "label": "Customer Region"}, + {"value": "order_priority", "label": "Order Priority"}, + {"value": "MONTHNAME(order_date)", "label": "Month"}, + {"value": "clerk", "label": "Clerk"} + ] + }, + { + "name": "order_filter_condition", + "usage": {"orders_filter_count": ["orders_filter"]}, + "type": "pydough", + "description": "Condition(s) by which orders can be filtered", + "options": [ + {"label": "Status O", "value": "(order_status == 'O')"}, + {"label": "Price above 3000", "value": "(total_price > 3000)"}, + {"label": "March", "value": "(MONTH(order_date) == 3)"}, + {"label": "High priority", "value": "ISIN(order_priority, ('1-URGENT', '2-HIGH'))"} + ] + }, + { + "name": "order_priority_levels", + "usage": {"order_lvl_priority": ["level"]}, + "type": "str", + "description": "Level of priority for an order", + "options": [ + {"label": "LEVEL 0", "value": "other"}, + {"label": "LEVEL 1", "value": "low"}, + {"label": "LEVEL 2", "value": "medium"}, + {"label": "LEVEL 3", "value": "high"} + ] + }, + { + "name": "dataframe_collection_column", + "usage": {"generate_df_collection": ["col1"]}, + "type": "list", + "description": "List with various types of colors to be used in the dataframe collection", + "options":[ + {"label": "COLORS", "value": "['blue', 'red', 'yellow', 'purple']"} + ] + }, + { + "name": "dataframe_collections_names", + "usage": {"generate_df_collection": ["collection_name"]}, + "type": "str", + "description": "Names of the dataframes to be created in the collection", + "options":[ + {"label": "NAME1", "value": "colors_collection"} + ] + } + ], + "definitions": [ + { + "name": "orders_filter_count", + "description": "Counts the number of orders placed that satisfied the given condition.", + "parameters": { + "orders_filter": { + "type": "pydough", + "description": "Condition(s) for the order to make it count" + } + }, + "source": "result = COUNT(orders.WHERE({1}))", + "answer_variable": "result" + }, + { + "name": "cumulative_orders_counter", + "description": "Calculates the cumulative counter of all orders placed from a base year through a given last year. Recursively sums the current year's counter and the cumulative counter of all next years.", + "parameters": { + "base_year": { "type": "int", "description": "The year through which to calculate cumulative revenue." }, + "last_year": { "type": "int", "description": "The earliest year to include in the cumulation (recursion base case)." } + }, + "source": "assert base_year <= last_year\nif {1} == {2}:\n result = COUNT(orders.WHERE(YEAR(order_date) == base_year))\nelse:\n result = (\n COUNT(orders.WHERE(YEAR(order_date) == base_year)) + cumulative_orders_counter(base_year + 1, last_year))\n", + "answer_variable": "result" + }, + { + "name": "order_revenue", + "description": "Calculates the revenue of an order, which is the sum of the extended price * (1 - discount) for all line items in the order.", + "parameters": {}, + "source": "result = SUM(lines.extended_price * (1 - lines.discount))\n", + "answer_variable": "result" + }, + { + "name": "orders_revenue_by", + "description": "Calculates the revenue of orders in a given year, partitioned by a specified dimension.", + "parameters": { + "arg_year": { + "type": "int", + "description": "The year for which to calculate the revenue of orders." + }, + "arg_dimension": { + "type": "pydough", + "description": "The dimension by which to partition the revenue calculation." + } + }, + "source": "result = orders.WHERE(({1} == YEAR(order_date))).CALCULATE(revenue=order_revenue(), dimension=({2})).PARTITION(name=\"orders_groups\", by=dimension).CALCULATE(dimension, segment_revenue=SUM(orders.revenue))\n", + "answer_variable": "result" + }, + { + "name": "top_bottom_comparison", + "description": "Compares the top and bottom groups of a partitioned orders", + "parameters": { + "arg_partitioned_orders": { + "type": "pydough", + "description": "The partitioned orders to compare." + }, + "arg_calculation": { + "type": "pydough", + "description": "The metric by which to compare the groups." + } + }, + "source": "result = {1}.CALCULATE(dimension, segment_revenue, comparison_value={2}).WHERE(ABSENT(PREV(segment_revenue, by=segment_revenue.DESC())) | ABSENT(NEXT(segment_revenue, by=segment_revenue.DESC())) )", + "answer_variable": "result" + }, + { + "name": "multiply_by_2", + "description": "Receives an integer and return its multiplication by 2.", + "parameters": { + "base_number": { + "type": "int", + "description": "Number being multiply by 2" + } + }, + "source": "result = {1} * 2\n", + "answer_variable": "result" + }, + { + "name": "order_lvl_priority", + "description": "Returns a list with orders priority based on the given level. High, Medium, Low", + "parameters": { + "level": { + "type": "str", + "description": "Level of the priority" + } + }, + "source": "if {1} == 'high':\n result = ['1-URGENT', '2-HIGH']\nelif {1} == 'medium':\n result = ['3-MEDIUM', '5-LOW']\nelse:\n result = ['4-NOT SPECIFIED']", + "answer_variable": "result" + }, + { + "name": "customer_calculate", + "description": "Returns a dictionary, representing the fields for CALCULATE a customer", + "parameters": { + "cust_name": { + "type": "str", + "description": "Rename for the customer name column" + }, + "cust_nation": { + "type": "str", + "description": "Rename for the customer nation name column" + }, + "balance": { + "type": "str", + "description": "Rename for the customer account balance column" + } + }, + "source": "result = {\n {1}: name,\n {2}: nation.name,\n {3}: account_balance\n}", + "answer_variable": "result" + }, + { + "name": "generate_range_collection", + "description": "Generates a simple range collection and returns it", + "parameters":{ + "start": { + "type": "int", + "description": "Start of the range collection" + }, + "end": { + "type": "int", + "description": "End of the range collection" + } + }, + "source": "result = pydough.range_collection('template_range', 'idx', {1}, {2})", + "answer_variable": "result" + }, + { + "name": "range_cross_collection", + "description": "Generates a range collection and cross it with the given collection", + "parameters":{ + "cross_collection": { + "type": "pydough", + "description": "Pydough collection to cross with the generated range collection" + }, + "range_name" : { + "type": "str", + "description": "Name for the generated range collection" + }, + "start": { + "type": "int", + "description": "Start of the range collection" + }, + "end": { + "type": "int", + "description": "End of the range collection" + }, + "cross_cond":{ + "type": "pydough", + "description": "Condition for the cross operation" + } + }, + "source": "created_range = pydough.range_collection({2}, 'idx', {3}, {4}).CALCULATE(idx)\nresult = created_range.CROSS({1}).WHERE({5})", + "answer_variable": "result" + }, + { + "name": "generate_df_collection", + "description": "Generates a dataframe collection from the given parameters", + "parameters": { + "collection_name": { + "type": "str", + "description": "Name for the generated dataframe collection" + }, + "col1": { + "type": "list", + "description": "List with the data for the column of the new dataframe" + } + }, + "source": "assert len(col1) > 0\nnew_df = pd.DataFrame({\n'names': {2}, 'idx': range(len(col1))})\nresult = pydough.dataframe_collection(name={1}, dataframe=new_df, unique_column_names=['idx'])", + "answer_variable": "result" + }, + { + "name": "dataframe_input_collection", + "description": "Generates a dataframe collection from the given parameters", + "parameters": { + "collection_name": { + "type": "str", + "description": "Name for the generated dataframe collection" + }, + "new_df": { + "type": "pd.DataFrame", + "description": "New dataframe to create the collection with" + }, + "unique_columns": { + "type": "list", + "description": "List of unique column names for the dataframe collection" + } + }, + "source": "result = pydough.dataframe_collection({1}, {2}, {3})", + "answer_variable": "result" + }, + { + "name": "temporary_nations", + "description": "Generates a temporary table with nations filtered by the given region", + "parameters": { + "filtered_region": { + "type": "str", + "description": "Region to filter the nations by" + } + }, + "source": "my_nations = nations.WHERE(region.name == {1})\nnations_tmp = pydough.to_table(my_nations, name='region_nations_t1', write_path='E2E_TESTS_DB.PUBLIC.region_nations_t1', temp=True, replace=True)\nresult = nations_tmp.CALCULATE(name)", + "answer_variable": "result" + }, + { + "name": "add_datetime_days", + "description": "Adds a specified number of days to a base datetime and returns the resulting datetime", + "parameters": { + "base_datetime": { + "type": "datetime", + "description": "Base datetime to add days to" + }, + "adding_days": { + "type": "int", + "description": "Number of days to add" + } + }, + "source": "result = DATETIME({1}, f'+{{2}} days')", + "answer_variable": "result" + }, + { + "name": "add_datetime_months", + "description": "Adds a specified number of months to a base datetime and returns the resulting datetime", + "parameters": { + "base_datetime": { + "type": "datetime", + "description": "Base datetime to add months to" + }, + "adding_months": { + "type": "int", + "description": "Number of months to add" + } + }, + "source": "result = pd.to_datetime({1}) + pd.DateOffset(months={2})", + "answer_variable": "result" + } + ] + }, "additional definitions": [], "verified pydough analysis": [], "extra semantic info": {} diff --git a/tests/test_metadata/trino_graphs.json b/tests/test_metadata/trino_graphs.json index f6617ebc4..53e11afbe 100644 --- a/tests/test_metadata/trino_graphs.json +++ b/tests/test_metadata/trino_graphs.json @@ -845,6 +845,312 @@ "synonyms": ["transactions", "purchases"] } ], + "templates": { + "attributes": [ + { + "name": "years", + "usage": {"orders_revenue_by": ["arg_year"], "multiply_by_2": ["base_number"]}, + "type": "int", + "description": "The year for which to calculate the revenue of orders.", + "options": [ + { "label": "Year 1992", "value": 1992}, + { "label": "Year 1993", "value": 1993}, + { "label": "Year 1994", "value": 1994}, + { "label": "Year 1995", "value": 1995}, + { "label": "Year 1996", "value": 1996}, + { "label": "Year 1997", "value": 1997}, + { "label": "Year 1998", "value": 1998} + ] + }, + { + "name": "order_dimensions", + "usage": {"orders_revenue_by": ["arg_dimension"]}, + "type": "pydough", + "description": "The dimension by which to partition the orders calculation.", + "options": [ + {"value": "customer.market_segment", "label": "Customer Market Segment"}, + {"value": "customer.nation.name", "label": "Customer Nation"}, + {"value": "customer.nation.region.name", "label": "Customer Region"}, + {"value": "order_priority", "label": "Order Priority"}, + {"value": "MONTHNAME(order_date)", "label": "Month"}, + {"value": "clerk", "label": "Clerk"} + ] + }, + { + "name": "order_filter_condition", + "usage": {"orders_filter_count": ["orders_filter"]}, + "type": "pydough", + "description": "Condition(s) by which orders can be filtered", + "options": [ + {"label": "Status O", "value": "(order_status == 'O')"}, + {"label": "Price above 3000", "value": "(total_price > 3000)"}, + {"label": "March", "value": "(MONTH(order_date) == 3)"}, + {"label": "High priority", "value": "ISIN(order_priority, ('1-URGENT', '2-HIGH'))"} + ] + }, + { + "name": "order_priority_levels", + "usage": {"order_lvl_priority": ["level"]}, + "type": "str", + "description": "Level of priority for an order", + "options": [ + {"label": "LEVEL 0", "value": "other"}, + {"label": "LEVEL 1", "value": "low"}, + {"label": "LEVEL 2", "value": "medium"}, + {"label": "LEVEL 3", "value": "high"} + ] + }, + { + "name": "dataframe_collection_column", + "usage": {"generate_df_collection": ["col1"]}, + "type": "list", + "description": "List with various types of colors to be used in the dataframe collection", + "options":[ + {"label": "COLORS", "value": "['blue', 'red', 'yellow', 'purple']"} + ] + }, + { + "name": "dataframe_collections_names", + "usage": {"generate_df_collection": ["collection_name"]}, + "type": "str", + "description": "Names of the dataframes to be created in the collection", + "options":[ + {"label": "NAME1", "value": "colors_collection"} + ] + } + ], + "definitions": [ + { + "name": "orders_filter_count", + "description": "Counts the number of orders placed that satisfied the given condition.", + "parameters": { + "orders_filter": { + "type": "pydough", + "description": "Condition(s) for the order to make it count" + } + }, + "source": "result = COUNT(orders.WHERE({1}))", + "answer_variable": "result" + }, + { + "name": "cumulative_orders_counter", + "description": "Calculates the cumulative counter of all orders placed from a base year through a given last year. Recursively sums the current year's counter and the cumulative counter of all next years.", + "parameters": { + "base_year": { "type": "int", "description": "The year through which to calculate cumulative revenue." }, + "last_year": { "type": "int", "description": "The earliest year to include in the cumulation (recursion base case)." } + }, + "source": "assert base_year <= last_year\nif {1} == {2}:\n result = COUNT(orders.WHERE(YEAR(order_date) == base_year))\nelse:\n result = (\n COUNT(orders.WHERE(YEAR(order_date) == base_year)) + cumulative_orders_counter(base_year + 1, last_year))\n", + "answer_variable": "result" + }, + { + "name": "order_revenue", + "description": "Calculates the revenue of an order, which is the sum of the extended price * (1 - discount) for all line items in the order.", + "parameters": {}, + "source": "result = SUM(lines.extended_price * (1 - lines.discount))\n", + "answer_variable": "result" + }, + { + "name": "orders_revenue_by", + "description": "Calculates the revenue of orders in a given year, partitioned by a specified dimension.", + "parameters": { + "arg_year": { + "type": "int", + "description": "The year for which to calculate the revenue of orders." + }, + "arg_dimension": { + "type": "pydough", + "description": "The dimension by which to partition the revenue calculation." + } + }, + "source": "result = orders.WHERE(({1} == YEAR(order_date))).CALCULATE(revenue=order_revenue(), dimension=({2})).PARTITION(name=\"orders_groups\", by=dimension).CALCULATE(dimension, segment_revenue=SUM(orders.revenue))\n", + "answer_variable": "result" + }, + { + "name": "top_bottom_comparison", + "description": "Compares the top and bottom groups of a partitioned orders", + "parameters": { + "arg_partitioned_orders": { + "type": "pydough", + "description": "The partitioned orders to compare." + }, + "arg_calculation": { + "type": "pydough", + "description": "The metric by which to compare the groups." + } + }, + "source": "result = {1}.CALCULATE(dimension, segment_revenue, comparison_value={2}).WHERE(ABSENT(PREV(segment_revenue, by=segment_revenue.DESC())) | ABSENT(NEXT(segment_revenue, by=segment_revenue.DESC())) )", + "answer_variable": "result" + }, + { + "name": "multiply_by_2", + "description": "Receives an integer and return its multiplication by 2.", + "parameters": { + "base_number": { + "type": "int", + "description": "Number being multiply by 2" + } + }, + "source": "result = {1} * 2\n", + "answer_variable": "result" + }, + { + "name": "order_lvl_priority", + "description": "Returns a list with orders priority based on the given level. High, Medium, Low", + "parameters": { + "level": { + "type": "str", + "description": "Level of the priority" + } + }, + "source": "if {1} == 'high':\n result = ['1-URGENT', '2-HIGH']\nelif {1} == 'medium':\n result = ['3-MEDIUM', '5-LOW']\nelse:\n result = ['4-NOT SPECIFIED']", + "answer_variable": "result" + }, + { + "name": "customer_calculate", + "description": "Returns a dictionary, representing the fields for CALCULATE a customer", + "parameters": { + "cust_name": { + "type": "str", + "description": "Rename for the customer name column" + }, + "cust_nation": { + "type": "str", + "description": "Rename for the customer nation name column" + }, + "balance": { + "type": "str", + "description": "Rename for the customer account balance column" + } + }, + "source": "result = {\n {1}: name,\n {2}: nation.name,\n {3}: account_balance\n}", + "answer_variable": "result" + }, + { + "name": "generate_range_collection", + "description": "Generates a simple range collection and returns it", + "parameters":{ + "start": { + "type": "int", + "description": "Start of the range collection" + }, + "end": { + "type": "int", + "description": "End of the range collection" + } + }, + "source": "result = pydough.range_collection('template_range', 'idx', {1}, {2})", + "answer_variable": "result" + }, + { + "name": "range_cross_collection", + "description": "Generates a range collection and cross it with the given collection", + "parameters":{ + "cross_collection": { + "type": "pydough", + "description": "Pydough collection to cross with the generated range collection" + }, + "range_name" : { + "type": "str", + "description": "Name for the generated range collection" + }, + "start": { + "type": "int", + "description": "Start of the range collection" + }, + "end": { + "type": "int", + "description": "End of the range collection" + }, + "cross_cond":{ + "type": "pydough", + "description": "Condition for the cross operation" + } + }, + "source": "created_range = pydough.range_collection({2}, 'idx', {3}, {4}).CALCULATE(idx)\nresult = created_range.CROSS({1}).WHERE({5})", + "answer_variable": "result" + }, + { + "name": "generate_df_collection", + "description": "Generates a dataframe collection from the given parameters", + "parameters": { + "collection_name": { + "type": "str", + "description": "Name for the generated dataframe collection" + }, + "col1": { + "type": "list", + "description": "List with the data for the column of the new dataframe" + } + }, + "source": "assert len(col1) > 0\nnew_df = pd.DataFrame({\n'names': {2}, 'idx': range(len(col1))})\nresult = pydough.dataframe_collection(name={1}, dataframe=new_df, unique_column_names=['idx'])", + "answer_variable": "result" + }, + { + "name": "dataframe_input_collection", + "description": "Generates a dataframe collection from the given parameters", + "parameters": { + "collection_name": { + "type": "str", + "description": "Name for the generated dataframe collection" + }, + "new_df": { + "type": "pd.DataFrame", + "description": "New dataframe to create the collection with" + }, + "unique_columns": { + "type": "list", + "description": "List of unique column names for the dataframe collection" + } + }, + "source": "result = pydough.dataframe_collection({1}, {2}, {3})", + "answer_variable": "result" + }, + { + "name": "temporary_nations", + "description": "Generates a temporary table with nations filtered by the given region", + "parameters": { + "filtered_region": { + "type": "str", + "description": "Region to filter the nations by" + } + }, + "source": "my_nations = nations.WHERE(region.name == {1})\nnations_tmp = pydough.to_table(my_nations, name='region_nations_t1', as_view=True, write_path='memory.default.region_nations_t1',temp=False, replace=True)\nresult = nations_tmp.CALCULATE(name)", + "answer_variable": "result" + }, + { + "name": "add_datetime_days", + "description": "Adds a specified number of days to a base datetime and returns the resulting datetime", + "parameters": { + "base_datetime": { + "type": "datetime", + "description": "Base datetime to add days to" + }, + "adding_days": { + "type": "int", + "description": "Number of days to add" + } + }, + "source": "result = DATETIME({1}, f'+{{2}} days')", + "answer_variable": "result" + }, + { + "name": "add_datetime_months", + "description": "Adds a specified number of months to a base datetime and returns the resulting datetime", + "parameters": { + "base_datetime": { + "type": "datetime", + "description": "Base datetime to add months to" + }, + "adding_months": { + "type": "int", + "description": "Number of months to add" + } + }, + "source": "result = pd.to_datetime({1}) + pd.DateOffset(months={2})", + "answer_variable": "result" + } + ] + }, "additional definitions": [ "Revenue for a lineitem is the extended_price * (1 - discount) * (1 - tax) minus quantity * supply_cost from the corresponding supply record", "A domestic shipment is a lineitem where the customer and supplier are from the same nation", diff --git a/tests/test_pipeline_tpch_custom.py b/tests/test_pipeline_tpch_custom.py index 83f57c333..113ec066e 100644 --- a/tests/test_pipeline_tpch_custom.py +++ b/tests/test_pipeline_tpch_custom.py @@ -195,10 +195,6 @@ year_month_nation_orders, yoy_change_in_num_orders, ) -from tests.test_pydough_functions.tpch_templates import ( - template_api, - template_call, -) from tests.test_pydough_functions.user_collections import ( dataframe_collection_bad_1, dataframe_collection_bad_2, @@ -5212,153 +5208,6 @@ ), id="monthname_function_1", ), - pytest.param( - PyDoughPandasTest( - "result = orders.WHERE(" - "(1996 == YEAR(order_date))\n" - ").CALCULATE(\n" - "revenue=order_revenue()\n" - ")\n", - "TPCH", - lambda: pd.DataFrame({}), - "template_test", - ), - id="template_test", - ), - pytest.param( - PyDoughPandasTest( - "result = orders_revenue_by(1996, customer.market_segment).WHERE(" - "(dimension == 'FURNITURE')\n" - ")\n", - "TPCH", - lambda: pd.DataFrame( - {"dimension": ["FURNITURE"], "segment_revenue": [6.671056e09]} - ), - "nested_template", - ), - id="nested_template", - ), - pytest.param( - PyDoughPandasTest( - "selected_customers = customers.WHERE(" - "(account_balance >= template_literal(9000))\n" - ")\n" - "result = TPCH.CALCULATE(n_custs=COUNT(selected_customers))\n", - "TPCH", - lambda: pd.DataFrame({"n_custs": [13533]}), - "literal_template", - ), - id="literal_template", - ), - pytest.param( - PyDoughPandasTest( - "result = pydough.call_template(\n" - " 'orders_revenue_by', {'arg_year': 'Year 1998', 'arg_dimension': 'Customer Market Segment'}" - ")\n", - "TPCH", - lambda: pd.DataFrame( - { - "dimension": [ - "AUTOMOBILE", - "BUILDING", - "FURNITURE", - "HOUSEHOLD", - "MACHINERY", - ], - "segment_revenue": [ - 3.839372e09, - 3.943587e09, - 3.897561e09, - 3.877929e09, - 3.875373e09, - ], - } - ), - "api_template_call_simple", - ), - id="api_template_call_simple", - ), - pytest.param( - PyDoughPandasTest( - "selected_customers = customers.WHERE(" - " (account_balance >= pydough.call_template('template_literal', {'base_number': 'Year 1992'}))\n" - ")\n" - "result = TPCH.CALCULATE(n_custs=COUNT(selected_customers))\n", - "TPCH", - lambda: pd.DataFrame( - { - "dimension": [ - "AUTOMOBILE", - "BUILDING", - "FURNITURE", - "HOUSEHOLD", - "MACHINERY", - ], - "segment_revenue": [ - 3.839372e09, - 3.943587e09, - 3.897561e09, - 3.877929e09, - 3.875373e09, - ], - } - ), - "api_template_call_literal", - ), - id="api_template_call_literal", - ), - pytest.param( - PyDoughPandasTest( - template_call, - "TPCH", - lambda: pd.DataFrame( - { - "dimension": [ - "AUTOMOBILE", - "BUILDING", - "FURNITURE", - "HOUSEHOLD", - "MACHINERY", - ], - "segment_revenue": [ - 6563815859.6161, - 6752822196.8331995, - 6671055612.9225, - 6670709849.0533, - 6620124465.2663, - ], - } - ), - "template_call_func", - ), - id="template_call_func", - ), - pytest.param( - PyDoughPandasTest( - template_api, - "TPCH", - lambda: pd.DataFrame( - { - "dimension": [ - "AFRICA", - "AMERICA", - "ASIA", - "EUROPE", - "MIDDLE EAST", - ], - "segment_revenue": [ - 6.612832e09, - 6.618720e09, - 6.688478e09, - 6.746079e09, - 6.612420e09, - ], - } - ), - "template_api_call", - ), - id="template_api_call", - ), ], ) def tpch_custom_pipeline_test_data(request) -> PyDoughPandasTest: diff --git a/tests/test_pydough_functions/tpch_templates.py b/tests/test_pydough_functions/tpch_templates.py index 27fb879bc..e48b4d97c 100644 --- a/tests/test_pydough_functions/tpch_templates.py +++ b/tests/test_pydough_functions/tpch_templates.py @@ -7,14 +7,116 @@ # ruff & mypy should not try to typecheck or verify any of this import pydough +import pandas as pd -def template_call(): - return orders_revenue_by(1996, customer.market_segment) +def template_simple_call(): + # Top 5 customers with most orders from 1996 where price is greater then 3000 + return customers.CALCULATE( + key, + n_orders=orders_filter_count((total_price > 3000) & (YEAR(order_date) == 1996)), + ).TOP_K(5, by=(n_orders.DESC(), key.ASC())) -def template_api(): - return pydough.call_template( - "orders_revenue_by", - labels={"arg_year": "Year 1996", "arg_dimension": "Customer Region"}, +def template_api_simple_call(): + # Which nations has the most customers with orders above 3000 + selected_orders = pydough.call_template( + "orders_filter_count", labels={"orders_filter": "Price above 3000"} + ) + return nations.CALCULATE( + name, n_customers=COUNT(customers.WHERE(selected_orders > 0)) + ).TOP_K(3, by=(n_customers.DESC(), name.ASC())) + + +def template_follow_up_call(): + # Top/bottom segment comparison + orders_segmentation = pydough.call_template( + "orders_revenue_by", labels={"arg_year": "Year 1997", "arg_dimension": "Month"} + ) + + return top_bottom_comparison(orders_segmentation, AVG(orders.revenue)) + + +def template_literal_1(): + + min_account_balance = pydough.call_template( + "multiply_by_2", labels={"base_number": "Year 1992"} + ) + + selected_customers = customers.WHERE((account_balance >= min_account_balance)) + return TPCH.CALCULATE(n_custs=COUNT(selected_customers)) + + +def template_recursive_call(): + + return TPCH.CALCULATE( + y_1994=cumulative_orders_counter(1994, 1994), + y_1994_1996=cumulative_orders_counter(1994, 1996), + y_1994_1998=cumulative_orders_counter(1994, 1998), + ) + + +def template_cross_collection(): + selected_regions = regions.WHERE(MONOTONIC(1, key, 5)).CALCULATE(key, name) + + cross_collection = range_cross_collection( + selected_regions, "new_range", 3, 9, (key * 2 == idx) + ) + + return cross_collection.CALCULATE(idx, key, name) + + +def template_dataframe_collection(): + # Template that creates a dataframe collection + + generated_collection = pydough.call_template( + "generate_df_collection", labels={"collection_name": "NAME1", "col1": "COLORS"} + ) + + return generated_collection + + +def template_df_collection_df(): + # Template that receives a df, build a df collection and cross it with orders + + input_df = pd.DataFrame( + { + "cust_id": [1, 2, 3], + "customer_name": ["customer_1", "customer_2", "customer_3"], + } + ) + + df_collection = dataframe_input_collection( + "customers_collection", input_df, ["cust_id"] + ) + + selected_orders = orders.WHERE(ISIN(key, (1, 2, 3))).CALCULATE(key, clerk) + + return ( + df_collection.CALCULATE(cust_id, customer_name) + .CROSS(selected_orders) + .WHERE(cust_id == key) + .CALCULATE(cust_id, customer_name, key, clerk) + ) + + +def template_datetime_days(): + + base_date = pd.to_datetime("1996-12-01") + + return ( + orders.WHERE((order_date == DATETIME(base_date))) + .CALCULATE(key, order_date, date_plus_days=add_datetime_days(base_date, 10)) + .TOP_K(5, by=key.ASC()) + ) + + +def template_datetime_months(): + + base_date = pd.to_datetime("1995-10-11") + + return ( + orders.WHERE((order_date == DATETIME(base_date))) + .CALCULATE(key, order_date, date_plus_months=add_datetime_months(base_date, 10)) + .TOP_K(5, by=key.ASC()) ) diff --git a/tests/test_templates.py b/tests/test_templates.py new file mode 100644 index 000000000..445e12ee0 --- /dev/null +++ b/tests/test_templates.py @@ -0,0 +1,435 @@ +""" +TODO +""" + +import pandas as pd +import pytest + +from pydough.database_connectors.database_connector import ( + DatabaseContext, + DatabaseDialect, +) +from pydough.errors.error_types import PyDoughMetadataException, PyDoughTypeException +from pydough.metadata.graphs.graph_metadata import GraphMetadata +from pydough.metadata.parse import parse_json_metadata_from_file +from tests.test_pydough_functions.tpch_templates import ( + template_api_simple_call, + template_cross_collection, + template_dataframe_collection, + template_datetime_days, + template_datetime_months, + template_df_collection_df, + template_follow_up_call, + template_literal_1, + template_recursive_call, + template_simple_call, +) +from tests.testing_utilities import PyDoughPandasTest + + +@pytest.fixture( + params=[ + pytest.param( + # Test templates, simple direct call and using from_string + PyDoughPandasTest( + "result = customers.CALCULATE(\n" + " key," + " n_orders=orders_filter_count(HAS(lines.WHERE(part.brand == 'Brand#45')))\n" + ").TOP_K(5, by=(n_orders.DESC(), key.ASC()))\n", + "TPCH", + lambda: pd.DataFrame( + { + "key": [102361, 126886, 33445, 43090, 67108], + "n_orders": [13, 12, 11, 11, 11], + } + ), + "templates_simple_call", + ), + id="templates_simple_call", + ), + pytest.param( + # Test templates, simple call using the api and from_string + PyDoughPandasTest( + "result = customers.CALCULATE(\n" + " key," + " n_orders=pydough.call_template('orders_filter_count', labels={'orders_filter': 'High priority'})\n" + ").TOP_K(5, by=(n_orders.DESC(), key.ASC()))\n", + "TPCH", + lambda: pd.DataFrame( + { + "key": [15859, 75160, 82531, 94393, 99070], + "n_orders": [21, 21, 21, 21, 21], + } + ), + "templates_simple_call_api", + ), + id="templates_simple_call_api", + ), + pytest.param( + # Test templates, simple call using test function and directly + PyDoughPandasTest( + template_simple_call, + "TPCH", + lambda: pd.DataFrame( + { + "key": [133759, 43645, 1183, 4219, 7981], + "n_orders": [13, 12, 11, 11, 11], + } + ), + "templates_simple_call_func", + ), + id="templates_simple_call_func", + ), + pytest.param( + # Test templates, simple call using test function and api + PyDoughPandasTest( + template_api_simple_call, + "TPCH", + lambda: pd.DataFrame( + { + "name": ["FRANCE", "RUSSIA", "ROMANIA"], + "n_customers": [4149, 4089, 4087], + } + ), + "templates_simple_call_func_api", + ), + id="templates_simple_call_func_api", + ), + pytest.param( + # Test templates, template called inside another template, both + # directly + PyDoughPandasTest( + "result = orders_revenue_by(1996, customer.market_segment).WHERE(" + "(dimension == 'FURNITURE')\n" + ")\n", + "TPCH", + lambda: pd.DataFrame( + {"dimension": ["FURNITURE"], "segment_revenue": [6.671056e09]} + ), + "templates_nested_call", + ), + id="templates_nested_call", + ), + pytest.param( + # Test templates, two templates calls, first returns the input for the + # next one. First, called through api then second one directly. In a + # test function + PyDoughPandasTest( + template_follow_up_call, + "TPCH", + lambda: pd.DataFrame( + { + "dimension": ["May", "Feb"], + "segment_revenue": [2.841049e09, 2.563501e09], + "comparison_value": [145553.014750, 145786.015957], + } + ), + "templates_follow_up_call", + ), + id="templates_follow_up_call", + ), + pytest.param( + # Test templates, direct called and from_string, template that returns a literal + # integer + PyDoughPandasTest( + "selected_customers = customers.WHERE(" + "(account_balance >= multiply_by_2(1000))\n" + ")\n" + "result = TPCH.CALCULATE(n_custs=COUNT(selected_customers))\n", + "TPCH", + lambda: pd.DataFrame({"n_custs": [109077]}), + "templates_literal", + ), + id="templates_literal", + ), + pytest.param( + # Test templates literals using api and calling a test function, template + # that returns an integer + PyDoughPandasTest( + template_literal_1, + "TPCH", + lambda: pd.DataFrame({"n_custs": [81779]}), + "templates_literal_api_func", + ), + id="templates_literal_api_func", + ), + pytest.param( + # Test templates literals, template that generates a list of + # literals and returns it called through the pydough API and from_string + PyDoughPandasTest( + "high_priority_list = pydough.call_template('order_lvl_priority', labels={'level': 'LEVEL 3'})\n" + "medium_priority_list = pydough.call_template('order_lvl_priority', labels={'level': 'LEVEL 2'})\n" + "low_priority_list = pydough.call_template('order_lvl_priority', labels={'level': 'LEVEL 1'})\n" + "other_priority_list = pydough.call_template('order_lvl_priority', labels={'level': 'LEVEL 0'})\n" + "result = TPCH.CALCULATE(\n" + " n_high_orders=COUNT(orders.WHERE(ISIN(order_priority, high_priority_list))),\n" + " n_medium_orders=COUNT(orders.WHERE(ISIN(order_priority, medium_priority_list))),\n" + " n_low_orders=COUNT(orders.WHERE(ISIN(order_priority, low_priority_list))),\n" + " n_other_orders=COUNT(orders.WHERE(ISIN(order_priority, other_priority_list)))\n" + ")", + "TPCH", + lambda: pd.DataFrame( + { + "n_high_orders": [600434], + "n_medium_orders": [599312], + "n_low_orders": [300254], + "n_other_orders": [300254], + } + ), + "templates_literal_list", + ), + id="templates_literal_list", + ), + pytest.param( + # Test templates literals, template that generates a dictionary of + # literals and returns it + PyDoughPandasTest( + "result = customers.CALCULATE(\n" + " **customer_calculate('full_name', 'country', 'customer_balance')\n" + ").TOP_K(5, by=customer_balance.DESC())", + "TPCH", + lambda: pd.DataFrame( + { + "full_name": [ + "Customer#000061453", + "Customer#000069321", + "Customer#000144232", + "Customer#000002487", + "Customer#000023828", + ], + "country": [ + "MOROCCO", + "MOROCCO", + "GERMANY", + "UNITED STATES", + "MOZAMBIQUE", + ], + "customer_balance": [ + 9999.99, + 9999.96, + 9999.74, + 9999.72, + 9999.64, + ], + } + ), + "templates_literal_dict", + ), + id="templates_literal_dict", + ), + pytest.param( + # Test templates datetime, receives a datetime and adds days to it using + # pydough + PyDoughPandasTest( + template_datetime_days, + "TPCH", + lambda: pd.DataFrame( + { + "key": [2, 19008, 23686, 57953, 63589], + "order_date": [ + "1996-12-01", + "1996-12-01", + "1996-12-01", + "1996-12-01", + "1996-12-01", + ], + "date_plus_days": [ + "1996-12-11", + "1996-12-11", + "1996-12-11", + "1996-12-11", + "1996-12-11", + ], + } + ), + "templates_literal_datetime_days", + ), + id="templates_literal_datetime_days", + ), + pytest.param( + # Test templates datetime, receives a datetime and adds months to it, + # returning a datetime + PyDoughPandasTest( + template_datetime_months, + "TPCH", + lambda: pd.DataFrame( + { + "key": [4, 2532, 7075, 9127, 36610], + "order_date": [ + "1995-10-11", + "1995-10-11", + "1995-10-11", + "1995-10-11", + "1995-10-11", + ], + "date_plus_months": [ + "1996-08-11 00:00:00", + "1996-08-11 00:00:00", + "1996-08-11 00:00:00", + "1996-08-11 00:00:00", + "1996-08-11 00:00:00", + ], + } + ), + "templates_literal_datetime_months", + ), + id="templates_literal_datetime_months", + ), + pytest.param( + # Test templates, template that generates pydough recursively + PyDoughPandasTest( + template_recursive_call, + "TPCH", + lambda: pd.DataFrame( + { + "y_1994": [227597], + "y_1994_1996": [684860], + "y_1994_1998": [1046266], + } + ), + "templates_recursion_func", + ), + id="templates_recursion_func", + ), + pytest.param( + # Test templates/range collection, template called directly creates + # a range collection and returns it + PyDoughPandasTest( + "result = generate_range_collection(1, 5)", + "TPCH", + lambda: pd.DataFrame({"idx": [1, 2, 3, 4]}), + "templates_range_collection", + ), + id="templates_range_collection", + ), + pytest.param( + # Test templates/range collection, template called directly creates + # a range collection and cross it with a given collection + PyDoughPandasTest( + template_cross_collection, + "TPCH", + lambda: pd.DataFrame( + { + "idx": [4, 6, 8], + "key": [2, 3, 4], + "name": ["ASIA", "EUROPE", "MIDDLE EAST"], + } + ), + "templates_range_collection_cross", + ), + id="templates_range_collection_cross", + ), + pytest.param( + # Test templates/dataframe collection, template called through the + # api and returns a dataframe collection + PyDoughPandasTest( + template_dataframe_collection, + "TPCH", + lambda: pd.DataFrame( + { + "names": ["blue", "red", "yellow", "purple"], + "idx": [0, 1, 2, 3], + } + ), + "templates_df_collection_api", + ), + id="templates_df_collection_api", + ), + pytest.param( + # Test dataframe collection, the templates receives a dataframe directly + # as input, and returns a dataframe collection, which is then used + # in a CROSS opeartion. + PyDoughPandasTest( + template_df_collection_df, + "TPCH", + lambda: pd.DataFrame( + { + "cust_id": [1, 2, 3], + "customer_name": ["customer_1", "customer_2", "customer_3"], + "key": [1, 2, 3], + "clerk": [ + "Clerk#000000951", + "Clerk#000000880", + "Clerk#000000955", + ], + } + ), + "templates_df_collection_df_input", + ), + id="templates_df_collection_df_input", + ), + pytest.param( + # Test using to_table inside a template + PyDoughPandasTest( + "result = temporary_nations('ASIA')", + "TPCH", + lambda: pd.DataFrame( + { + "name": ["INDIA", "INDONESIA", "JAPAN", "CHINA", "VIETNAM"], + } + ), + "templates_to_table", + ), + id="templates_to_table", + ), + ] +) +def tpch_templates_test_data(request) -> PyDoughPandasTest: + """ + Test data for e2e tests on templates using the TPC-H database. + Returns an instance of PyDoughPandasTest containing information about the + test. + """ + return request.param + + +@pytest.mark.execute +def test_pipeline_e2e_tpch_templates( + tpch_templates_test_data: PyDoughPandasTest, + all_dialects_tpch_db_context: tuple[DatabaseContext, GraphMetadata], +): + """ + Test executing the the template queries with TPC-H data from the original + code generation. + """ + db_context, graph = all_dialects_tpch_db_context + + # Skip BodoSQL, since checking all the custom tests with + # it would take too long. + if db_context.dialect == DatabaseDialect.BODOSQL: + pytest.skip("Skipping tpch template test for BodoSQL.") + + tpch_templates_test_data.run_e2e_test( + lambda _: graph, + db_context, + coerce_types=True, + atol=5e-3, + ) + + +@pytest.mark.parametrize( + "graph_name, error_message", + [ + # Attr with no definitions + pytest.param( + "NO_TEMPLATES_DEFINITIONS", + "graph 'NO_TEMPLATES_DEFINITIONS' must be a JSON object containing a field 'definitions' and field 'definitions' must be a JSON array", + id="missing_definitions", + ), + # Attr with no options + pytest.param( + "NO_ATTIBUTE_OPTIONS", + "graph 'NO_ATTIBUTE_OPTIONS' must be a JSON object containing a field 'options' and field 'options' must be a JSON array", + id="missing_options", + ), + ], +) +def test_invalid_metadata_templates( + invalid_templates_graph_path: str, graph_name: str, error_message: str +) -> None: + with pytest.raises( + (PyDoughMetadataException, PyDoughTypeException), match=error_message + ): + parse_json_metadata_from_file( + file_path=invalid_templates_graph_path, graph_name=graph_name + ) From d72061d1bb76eb184f8b9b766bf3c478dc58f802 Mon Sep 17 00:00:00 2001 From: john-sanchez31 Date: Mon, 7 Sep 2026 11:54:19 -0600 Subject: [PATCH 06/14] adding error test for e2e and metadata definitions, adding error messages and attribute restrictions [run ci][run dialects] --- pydough/errors/error_utils.py | 4 + pydough/errors/pydough_error_builder.py | 4 +- pydough/metadata/graphs/graph_metadata.py | 35 +- .../metadata/templates/attribute_metadata.py | 62 ++- .../metadata/templates/template_metadata.py | 53 +- pydough/unqualified/unqualified_node.py | 5 +- pydough/unqualified/unqualified_transform.py | 34 +- tests/test_metadata/invalid_templates.json | 451 +++++++++++++++++- tests/test_templates.py | 217 ++++++++- 9 files changed, 818 insertions(+), 47 deletions(-) diff --git a/pydough/errors/error_utils.py b/pydough/errors/error_utils.py index fd9f276a1..130c1ff3d 100644 --- a/pydough/errors/error_utils.py +++ b/pydough/errors/error_utils.py @@ -625,6 +625,10 @@ def error_message(self, error_name: str) -> str: is_string, NonEmptyListOf(is_string) ) +attributes_usage_predicate: PyDoughPredicate = PossiblyEmptyMapOf( + is_string, PossiblyEmptyListOf(is_string) +) + ################################################################################ # Extraction functions diff --git a/pydough/errors/pydough_error_builder.py b/pydough/errors/pydough_error_builder.py index 0da739bb5..dd5664545 100644 --- a/pydough/errors/pydough_error_builder.py +++ b/pydough/errors/pydough_error_builder.py @@ -256,7 +256,7 @@ def sql_call_conversion_error( ) def undefined_function_call( - self, node: "UnqualifiedNode", *args, **kwargs + self, node: "UnqualifiedNode", available_templates: list[str], *args, **kwargs ) -> PyDoughException: """ Creates an exception for when a function call is made on an unqualified @@ -281,7 +281,7 @@ def undefined_function_call( ): suggestions: list[str] = find_possible_name_matches( term_name=node._parcel[1], - candidates=set(node._parcel[0]._parcel[1]), + candidates=set(node._parcel[0]._parcel[1]) | set(available_templates), atol=2, rtol=0.1, min_names=3, diff --git a/pydough/metadata/graphs/graph_metadata.py b/pydough/metadata/graphs/graph_metadata.py index c53b12593..ec903f8a9 100644 --- a/pydough/metadata/graphs/graph_metadata.py +++ b/pydough/metadata/graphs/graph_metadata.py @@ -37,6 +37,20 @@ class GraphMetadata(AbstractMetadata): Fields allowed in the JSON object describing a graph. """ + ALLOWED_TYPES: set[str] = { + "str", + "int", + "float", + "list", + "dict", + "pydough", + "pd.DataFrame", # Type for pandas dataframe for Dataframe Collections + "datetime", # Type for datatimes + } + """ + Allowed types for template attributes and definitions in the graph. + """ + def __init__( self, name: str, @@ -225,11 +239,24 @@ def get_all_labels(self) -> dict[str, str]: Fetches all of the labels defined in the graph's template attributes. """ all_labels: dict[str, str] = {} - for attribute in self.templates_attributes.values(): - all_labels.update(dict.fromkeys(attribute.options.keys(), attribute.name)) + for name, attribute in self.templates_attributes.items(): + all_labels.update(dict.fromkeys(attribute.options.keys(), name)) return all_labels + def is_valid_data_type(self, type_str: str) -> bool: + """ + Validate that type_str is exactly one of the allowed type names. + + Args: + `type_str`: the type name to validate. + + Returns: + Bool: True if type_str is a valid type name, False otherwise. + """ + + return type_str.strip() == type_str and type_str in self.ALLOWED_TYPES + def add_template_attribute(self, new_attribute: AbstractMetadata) -> None: """ Adds a new attribute to the graph. @@ -253,7 +280,7 @@ def add_template_attribute(self, new_attribute: AbstractMetadata) -> None: if new_attribute.name in self.templates_attributes: if self.templates_attributes[new_attribute.name] == new_attribute: raise PyDoughMetadataException( - f"Already added {new_attribute.error_name} to {self.error_name}" + f"Already added {new_attribute.error_name}" ) raise PyDoughMetadataException( f"Duplicate attributes: {new_attribute.error_name} versus {self.templates_attributes[new_attribute.name].error_name}" @@ -286,7 +313,7 @@ def add_template_definition( if name in self.templates_definitions: if self.templates_definitions[name] == new_template: raise PyDoughMetadataException( - f"Already added {name} to {self.error_name}" + f"Already added {name!r} to {self.error_name}" ) raise PyDoughMetadataException( f"Duplicate templates: {name} versus {self.templates_definitions[name]}" diff --git a/pydough/metadata/templates/attribute_metadata.py b/pydough/metadata/templates/attribute_metadata.py index 81a0dcfcd..54027a37d 100644 --- a/pydough/metadata/templates/attribute_metadata.py +++ b/pydough/metadata/templates/attribute_metadata.py @@ -5,10 +5,13 @@ from pydough.errors.error_types import PyDoughMetadataException from pydough.errors.error_utils import ( HasType, + attributes_usage_predicate, extract_array, extract_integer, extract_object, extract_string, + is_integer, + is_string, ) from pydough.metadata.abstract_metadata import AbstractMetadata from pydough.metadata.graphs import GraphMetadata @@ -100,9 +103,6 @@ def add_attribute_option(self, label: str, value: str | int) -> None: """ Add an option to the list of options """ - if label in self.options: - raise ValueError(f"Duplicate option label: {label!r}") - self.options[label] = value @staticmethod @@ -135,7 +135,15 @@ def parse_from_json( attr_usage: dict[str, list[str]] = extract_object( attribute_json, "usage", error_name ) + attributes_usage_predicate.verify(attr_usage, error_name) + attr_type: str = extract_string(attribute_json, "type", error_name) + # Validate the type of the attribute + if not graph.is_valid_data_type(attr_type): + raise PyDoughMetadataException( + f"Invalid type {attr_type!r} for attribute {attribute_name!r} in graph {graph.name!r}. Must be one of: {sorted(graph.ALLOWED_TYPES)}" + ) + attr_desc: str = extract_string(attribute_json, "description", error_name) new_attribute: AttributeMetadata = AttributeMetadata( @@ -149,26 +157,46 @@ def parse_from_json( # Parse and add the options attr_options: list = extract_array(attribute_json, "options", error_name) - for option in attr_options: - label: str = extract_string(option, "label", error_name) - value: str | int + if len(attr_options) == 0: + raise PyDoughMetadataException( + f"Template attribute {attribute_name!r} in graph {graph.name!r} " + f"must have at least one option defined." + ) + + option_error_name: str = ( + f"Option in attribute {attribute_name!r} in graph {graph.name!r}" + ) - try: - if type(option["value"]) is str: - value = extract_string(option, "value", error_name) - else: - value = extract_integer(option, "value", error_name) - except PyDoughMetadataException: - raise PyDoughMetadataException("Option value must be string or integer") + # Determine whether all option values are consistently strings or consistently + # integers -- reject any mix, and reject any other type entirely. + option_values = [option.get("value") for option in attr_options] + all_strings = all(is_string.accept(v) for v in option_values) + all_integers = all(is_integer.accept(v) for v in option_values) + + if not (all_strings or all_integers): + raise PyDoughMetadataException( + f"{option_error_name} 'value' fields must be either all strings or " + f"all integers (not a mix, and no other type)." + ) + + for option in attr_options: + label: str = extract_string(option, "label", option_error_name) + value: str | int = ( + extract_string(option, "value", option_error_name) + if all_strings + else extract_integer(option, "value", option_error_name) + ) graph_labels: dict[str, str] = graph.get_all_labels() - if label in graph_labels: - raise ValueError( + if label in graph_labels or label in new_attribute.options: + by_attribute: str = ( + graph_labels[label] if label in graph_labels else attribute_name + ) + raise PyDoughMetadataException( f"Duplicate option label: {label!r} for attribute {attribute_name!r}. " - f"The label is already in use by attribute {graph_labels[label]!r} " + f"The label is already in use by attribute {by_attribute!r} " f"in graph {graph.name!r}." ) - new_attribute.add_attribute_option(label, value) graph.add_template_attribute(new_attribute) diff --git a/pydough/metadata/templates/template_metadata.py b/pydough/metadata/templates/template_metadata.py index e6a0680c4..528a6e894 100644 --- a/pydough/metadata/templates/template_metadata.py +++ b/pydough/metadata/templates/template_metadata.py @@ -10,7 +10,7 @@ from typing import Any from pydough.errors.error_types import PyDoughMetadataException -from pydough.errors.error_utils import extract_object, extract_string +from pydough.errors.error_utils import extract_object, extract_string, is_valid_name from pydough.metadata.abstract_metadata import AbstractMetadata from pydough.metadata.graphs.graph_metadata import GraphMetadata @@ -61,7 +61,7 @@ def __init__( self._answer_variable: str = answer_variable self._parameters: dict[str, TemplateParameter] = ( - TemplateMetadata.parse_parameters_from_json(parameters) + TemplateMetadata.parse_parameters_from_json(graph, parameters, name) ) self._template_callable: Callable = self.create_template_callable(graph) @@ -123,7 +123,7 @@ def path(self) -> str: @staticmethod def create_error_name(name: str, graph_error_name: str): - return f"template definition {name!r} in {graph_error_name}" + return f"Template definition {name!r} in {graph_error_name}" @staticmethod def parse_from_json(graph: GraphMetadata, name: str, definition_json: dict) -> None: @@ -143,12 +143,22 @@ def parse_from_json(graph: GraphMetadata, name: str, definition_json: dict) -> N `PyDoughMetadataException`: if the JSON does not meet the necessary structure properties. """ + error_name: str = TemplateMetadata.create_error_name(name, graph.error_name) + # Validates the name for the template + is_valid_name.verify(name, error_name) + description: str = extract_string( definition_json, "description", graph.error_name ) answer_variable: str = extract_string( definition_json, "answer_variable", graph.error_name ) + error_answer_var: str = ( + f"Answer variable {answer_variable!r} of template {name!r} " + f"in graph {graph.name!r}" + ) + is_valid_name.verify(answer_variable, error_answer_var) + source: str = extract_string(definition_json, "source", graph.error_name) kwargs: dict[str, dict] = extract_object( definition_json, "parameters", graph.error_name @@ -167,7 +177,9 @@ def parse_from_json(graph: GraphMetadata, name: str, definition_json: dict) -> N @staticmethod def parse_parameters_from_json( + graph: GraphMetadata, parameters_json: dict, + template_name: str, ) -> dict[str, TemplateParameter]: """ Parses a JSON object into the parameters for a template definition @@ -188,16 +200,24 @@ def parse_parameters_from_json( template_params: dict[str, TemplateParameter] = {} for param_name, arg in parameters_json.items(): - if param_name in template_params: - raise PyDoughMetadataException( - f"Already added {param_name} to the template's parameters" - ) + error_param_name: str = f"Parameter {param_name!r} in template {template_name!r} in graph {graph.name!r}" + is_valid_name.verify(param_name, error_param_name) param_type: str = extract_string( arg, "type", "All parameters must have type" ) + # Validate the type of the attribute + if not graph.is_valid_data_type(param_type): + raise PyDoughMetadataException( + f"Invalid type {param_type!r} for the parameter {param_name!r}" + f" of template {template_name!r} in graph {graph.name!r}." + f" Must be one of: {sorted(graph.ALLOWED_TYPES)}" + ) + param_description: str = extract_string( - arg, "description", "All parameters must have description" + arg, + "description", + f"All parameters must have description in {template_name!r}", ) new_param = TemplateParameter(param_name, param_type, param_description) @@ -221,7 +241,6 @@ def create_template_callable( Returns: The callable built from `source`, not yet invoked. """ - template_str: str = self.create_template_def() import pydough @@ -233,11 +252,23 @@ def create_template_callable( graph_name: str = "_graph" visitor = AddRootVisitor(graph_name, known_names) - tree: ast.AST = ast.parse(template_str) + try: + tree: ast.AST = ast.parse(template_str) + except SyntaxError as e: + raise PyDoughMetadataException( + f"Template definition {self.name!r} does not contain valid Python code: {e}" + ) from e + new_tree: ast.AST = ast.fix_missing_locations(visitor.visit(tree)) transformed_code: str = ast.unparse(new_tree) - compiled = compile(transformed_code, filename=f"<{self.name}>", mode="exec") + try: + compiled = compile(transformed_code, filename=f"<{self.name}>", mode="exec") + except (SyntaxError, ValueError) as e: + raise PyDoughMetadataException( + f"Internal error: failed to compile transformed template for " + f"{self.name!r}: {e}" + ) from e # `_graph`, `pydough` and pd are baked into the function's globals so that # they're available whenever the function is later called diff --git a/pydough/unqualified/unqualified_node.py b/pydough/unqualified/unqualified_node.py index 9b1672421..a40f6f5df 100644 --- a/pydough/unqualified/unqualified_node.py +++ b/pydough/unqualified/unqualified_node.py @@ -136,6 +136,7 @@ def __getitem__(self, key): def __call__(self, *args, **kwargs): + available_templates: list[str] = [] if pydough.active_session.metadata: metadata_templates: dict[str, TemplateMetadata] = ( pydough.active_session.metadata.templates_definitions @@ -145,8 +146,10 @@ def __call__(self, *args, **kwargs): if name in metadata_templates: return metadata_templates[name].template_callable(*args, **kwargs) + available_templates.extend(metadata_templates.keys()) + raise pydough.active_session.error_builder.undefined_function_call( - self, *args, **kwargs + self, available_templates, *args, **kwargs ) def __bool__(self): diff --git a/pydough/unqualified/unqualified_transform.py b/pydough/unqualified/unqualified_transform.py index 38f4968e9..918b73ba8 100644 --- a/pydough/unqualified/unqualified_transform.py +++ b/pydough/unqualified/unqualified_transform.py @@ -19,6 +19,7 @@ from pydough.configs import PyDoughSession from pydough.errors import PyDoughSessionException, PyDoughUnqualifiedException +from pydough.errors.error_utils import find_possible_name_matches from pydough.metadata import GraphMetadata from pydough.metadata.templates import AttributeMetadata, TemplateMetadata @@ -642,6 +643,28 @@ def call_template( if pydough.active_session.metadata is None: raise ValueError("No metadata loaded in the current active session") + if name not in pydough.active_session.metadata.templates_definitions: + available_templates: list[str] = list( + pydough.active_session.metadata.templates_definitions.keys() + ) + suggestions: list[str] = find_possible_name_matches( + term_name=name, + candidates=set(available_templates), + atol=2, + rtol=0.1, + min_names=3, + max_names=5, + insert_cost=0.5, + delete_cost=1.0, + substitution_cost=1.0, + capital_cost=0.1, + ) + error_message: str = f"PyDough template {name!r} doesn't exist." + if len(suggestions) > 0: + suggestions_str: str = ", ".join(suggestions) + error_message += f" Did you mean: {suggestions_str}?" + raise ValueError(error_message) + calling_template: TemplateMetadata = ( pydough.active_session.metadata.templates_definitions[name] ) @@ -658,6 +681,13 @@ def call_template( template_kwargs: dict[str, dict[str, str | int]] = {} for arg_name, label in labels.items(): + if arg_name not in calling_template.parameters: + template_params: list[str] = list(calling_template.parameters.keys()) + template_params_str: str = ", ".join(template_params) + + raise ValueError( + f"Template {name!r} doesn't have a paramater called {arg_name!r}. Did you mean: {template_params_str}" + ) # Use this arg type to check the option value arg_type: str = calling_template.parameters[arg_name].type @@ -672,7 +702,7 @@ def call_template( or arg_name not in attribute.usage[name] ): raise ValueError( - f"The attribute '{attr_name}' is not available for parameter '{arg_name}' on template '{name}'" + f"The label {label!r} is not available for parameter '{arg_name}' on template '{name}'" ) # Types must match @@ -686,7 +716,7 @@ def call_template( break if kwarg is None: - raise ValueError(f"Label {label} not found in any attribute's options") + raise ValueError(f"Label {label!r} not found in any attribute's options") template_kwargs[arg_name] = kwarg diff --git a/tests/test_metadata/invalid_templates.json b/tests/test_metadata/invalid_templates.json index fa50566b9..94a53e77d 100644 --- a/tests/test_metadata/invalid_templates.json +++ b/tests/test_metadata/invalid_templates.json @@ -1,6 +1,28 @@ [ { - "name": "NO_TEMPLATES_DEFINITIONS", + "name": "INVALID_ATTRIBUTE_NAME", + "version": "V2", + "relationships": [], + "collections": [], + "templates":{ + "attributes": [ + { + "name": 123, + "usage": {}, + "type": "str", + "description": "Generic description", + "options": [ + { "label": "LABEL 1", "value": "VALUE 1"}, + { "label": "LABEL 2", "value": "VALUE 2"}, + { "label": "LABEL 3", "value": "VALUE 3"} + ] + } + ], + "definitions":[] + } + }, + { + "name": "DUPPLICATED_ATTRIBUTE_NAME", "version": "V2", "relationships": [], "collections": [], @@ -10,18 +32,183 @@ "name": "attr1", "usage": {}, "type": "str", - "description": "Generic desctiption", + "description": "Generic description", "options": [ { "label": "LABEL 1", "value": "VALUE 1"}, { "label": "LABEL 2", "value": "VALUE 2"}, { "label": "LABEL 3", "value": "VALUE 3"} ] + }, + { + "name": "attr1", + "usage": {}, + "type": "str", + "description": "Generic description", + "options": [ + { "label": "LABEL 4", "value": "VALUE 4"}, + { "label": "LABEL 5", "value": "VALUE 5"}, + { "label": "LABEL 6", "value": "VALUE 6"} + ] } - ] + ], + "definitions":[] + } + }, + { + "name": "INVALID_ATTRIBUTE_USAGE", + "version": "V2", + "relationships": [], + "collections": [], + "templates":{ + "attributes": [ + { + "name": "attr1", + "usage": {"template_1": "invalid_usage"}, + "type": "str", + "description": "Generic description", + "options": [ + { "label": "LABEL 1", "value": "VALUE 1"}, + { "label": "LABEL 2", "value": "VALUE 2"}, + { "label": "LABEL 3", "value": "VALUE 3"} + ] + } + ], + "definitions":[] + } + }, + { + "name": "INVALID_ATTRIBUTE_TYPE", + "version": "V2", + "relationships": [], + "collections": [], + "templates":{ + "attributes": [ + { + "name": "attr1", + "usage": {}, + "type": "invalid_type", + "description": "Generic description", + "options": [ + { "label": "LABEL 1", "value": "VALUE 1"}, + { "label": "LABEL 2", "value": "VALUE 2"}, + { "label": "LABEL 3", "value": "VALUE 3"} + ] + } + ], + "definitions":[] + } + }, + { + "name": "NO_ATTRIBUTE_OPTIONS", + "version": "V2", + "relationships": [], + "collections": [], + "templates":{ + "attributes": [ + { + "name": "attr1", + "usage": {}, + "type": "str", + "description": "Generic description", + "options": [] + } + ], + "definitions":[] + } + }, + { + "name": "INVALID_ATTRIBUTE_OPTION_LABEL", + "version": "V2", + "relationships": [], + "collections": [], + "templates":{ + "attributes": [ + { + "name": "attr1", + "usage": {}, + "type": "str", + "description": "Generic description", + "options": [ + { "label": 123, "value": "VALUE 1"} + ] + } + ], + "definitions":[] + } + }, + { + "name": "INVALID_ATTRIBUTE_OPTION_VALUE", + "version": "V2", + "relationships": [], + "collections": [], + "templates":{ + "attributes": [ + { + "name": "attr1", + "usage": {}, + "type": "str", + "description": "Generic description", + "options": [ + { "label": "LABEL 1", "value": ["INVALID", "VALUE"]} + ] + } + ], + "definitions":[] + } + }, + { + "name": "DUPPLICATED_ATTRIBUTE_OPTION_LABEL", + "version": "V2", + "relationships": [], + "collections": [], + "templates":{ + "attributes": [ + { + "name": "attr1", + "usage": {}, + "type": "str", + "description": "Generic description", + "options": [ + { "label": "LABEL 1", "value": "VALUE 1"}, + { "label": "LABEL 1", "value": "VALUE 2"} + ] + } + ], + "definitions":[] + } + }, + { + "name": "DUPPLICATED_ATTRIBUTE_OPTION_LABEL_2", + "version": "V2", + "relationships": [], + "collections": [], + "templates":{ + "attributes": [ + { + "name": "attr1", + "usage": {}, + "type": "str", + "description": "Generic description", + "options": [ + { "label": "LABEL 1", "value": "VALUE 1"}, + { "label": "LABEL 2", "value": "VALUE 2"} + ] + }, + { + "name": "attr2", + "usage": {}, + "type": "str", + "description": "Generic description", + "options": [ + { "label": "LABEL 1", "value": "VALUE 1"} + ] + } + ], + "definitions":[] } }, { - "name": "NO_ATTIBUTE_OPTIONS", + "name": "MIXED_TYPE_ATTRIBUTE_OPTIONS", "version": "V2", "relationships": [], "collections": [], @@ -31,12 +218,266 @@ "name": "attr1", "usage": {}, "type": "str", - "description": "Generic desctiption", + "description": "Generic description", "options": [ + { "label": "LABEL 1", "value": "VALUE 1"}, + { "label": "LABEL 2", "value": 2} ] } ], "definitions":[] } + }, + { + "name": "NO_TEMPLATES_DEFINITIONS", + "version": "V2", + "relationships": [], + "collections": [], + "templates":{ + "attributes": [ + { + "name": "attr1", + "usage": {}, + "type": "str", + "description": "Generic description", + "options": [ + { "label": "LABEL 1", "value": "VALUE 1"}, + { "label": "LABEL 2", "value": "VALUE 2"}, + { "label": "LABEL 3", "value": "VALUE 3"} + ] + } + ] + } + }, + { + "name": "TEMPLATE_INVALID_NAME", + "version": "V2", + "relationships": [], + "collections": [], + "templates":{ + "definitions": [ + { + "name": "name with space", + "description": "Generic description.", + "parameters": { + "orders_filter": { + "type": "pydough", + "description": "Generic description" + } + }, + "source": "result = True", + "answer_variable": "result" + } + ] + } + }, + { + "name": "TEMPLATE_NAME_PYTHON_RESERVED_WORD", + "version": "V2", + "relationships": [], + "collections": [], + "templates":{ + "definitions": [ + { + "name": "continue", + "description": "Generic description.", + "parameters": { + "orders_filter": { + "type": "pydough", + "description": "Generic description" + } + }, + "source": "result = True", + "answer_variable": "result" + } + ] + } + }, + { + "name": "TEMPLATE_NAME_PYDOUGH_RESERVED_WORD", + "version": "V2", + "relationships": [], + "collections": [], + "templates":{ + "definitions": [ + { + "name": "CALCULATE", + "description": "Generic description.", + "parameters": { + "parameter_1": { + "type": "pydough", + "description": "Generic description" + } + }, + "source": "result = True", + "answer_variable": "result" + } + ] + } + }, + { + "name": "TEMPLATE_NAME_DUPLICATED", + "version": "V2", + "relationships": [], + "collections": [], + "templates":{ + "definitions": [ + { + "name": "template_1", + "description": "Generic description.", + "parameters": { + "parameter_1": { + "type": "pydough", + "description": "Generic description" + } + }, + "source": "result = True", + "answer_variable": "result" + }, + { + "name": "template_1", + "description": "Generic description.", + "parameters": { + "parameter_1": { + "type": "pydough", + "description": "Generic description" + } + }, + "source": "result = True", + "answer_variable": "result" + } + ] + } + }, + { + "name": "TEMPLATE_INVALID_PARAMETER_NAME", + "version": "V2", + "relationships": [], + "collections": [], + "templates":{ + "definitions": [ + { + "name": "template_1", + "description": "Generic description.", + "parameters": { + "invalid_#@param": { + "type": "invalid_type", + "description": "Generic description" + } + }, + "source": "result = True", + "answer_variable": "result" + } + ] + } + }, + { + "name": "TEMPLATE_INVALID_PARAMETER_TYPE", + "version": "V2", + "relationships": [], + "collections": [], + "templates":{ + "definitions": [ + { + "name": "template_1", + "description": "Generic description.", + "parameters": { + "parameter_1": { + "type": "invalid_type", + "description": "Generic description" + } + }, + "source": "result = True", + "answer_variable": "result" + } + ] + } + }, + { + "name": "TEMPLATE_PARAM_NO_DESC", + "version": "V2", + "relationships": [], + "collections": [], + "templates":{ + "definitions": [ + { + "name": "template_1", + "description": "Generic description.", + "parameters": { + "parameter_1": { + "type": "str" + } + }, + "source": "result = True", + "answer_variable": "result" + } + ] + } + }, + { + "name": "TEMPLATE_INVALID_ANSWER_VAR", + "version": "V2", + "relationships": [], + "collections": [], + "templates":{ + "definitions": [ + { + "name": "template_1", + "description": "Generic description.", + "parameters": { + "parameter_1": { + "type": "str", + "description": "Generic description" + } + }, + "source": "result = True", + "answer_variable": "(no_valid_identifier)" + } + ] + } + }, + { + "name": "TEMPLATE_INVALID_SOURCE_CODE", + "version": "V2", + "relationships": [], + "collections": [], + "templates":{ + "definitions": [ + { + "name": "template_1", + "description": "Generic description.", + "parameters": { + "parameter_1": { + "type": "str", + "description": "Generic description" + } + }, + "source": "+= no valid syntax", + "answer_variable": "result" + } + ] + } + }, + { + "name": "TEMPLATE_INVALID_SOURCE_CODE_2", + "version": "V2", + "relationships": [], + "collections": [], + "templates":{ + "definitions": [ + { + "name": "template_1", + "description": "Generic description.", + "parameters": { + "parameter_1": { + "type": "str", + "description": "Generic description" + } + }, + "source": "nonlocal foo", + "answer_variable": "result" + } + ] + } } ] \ No newline at end of file diff --git a/tests/test_templates.py b/tests/test_templates.py index 445e12ee0..f469a0579 100644 --- a/tests/test_templates.py +++ b/tests/test_templates.py @@ -2,6 +2,8 @@ TODO """ +import re + import pandas as pd import pytest @@ -24,7 +26,11 @@ template_recursive_call, template_simple_call, ) -from tests.testing_utilities import PyDoughPandasTest +from tests.testing_utilities import ( + PyDoughPandasTest, + graph_fetcher, + run_e2e_error_test, +) @pytest.fixture( @@ -407,20 +413,221 @@ def test_pipeline_e2e_tpch_templates( ) +@pytest.mark.execute +@pytest.mark.parametrize( + "pydough_impl, columns, error_message", + [ + pytest.param( + "result = orders_filter_count2()", + None, + "PyDough object orders_filter_count2 is not callable. Did you mean: orders_filter_count, RELCOUNT, STRCOUNT?", + id="unexisting_template_definition_call", + ), + pytest.param( + "result = pydough.call_template('orders_filter_count2', labels={})", + None, + "PyDough template 'orders_filter_count2' doesn't exist. Did you mean: orders_filter_count, order_revenue, order_lvl_priority?", + id="unexisting_template_definition_api", + ), + pytest.param( + "result = orders_filter_count(no_param=True)", + None, + re.escape( + "orders_filter_count() got an unexpected keyword argument 'no_param'" + ), + id="wrong_template_arguments_call", + ), + pytest.param( + "result = pydough.call_template('orders_filter_count', labels={'no_param': 'LABEL 1'})", + None, + "Template 'orders_filter_count' doesn't have a paramater called 'no_param'. Did you mean: orders_filter", + id="wrong_template_arguments_api", + ), + pytest.param( + "result = pydough.call_template('orders_filter_count', labels={'orders_filter': 'INVALID LABEL'})", + None, + "Label 'INVALID LABEL' not found in any attribute's options", + id="wrong_template_label_api", + ), + pytest.param( + "result = pydough.call_template('orders_filter_count', labels={'orders_filter': 'LEVEL 0'})", + None, + "The label 'LEVEL 0' is not available for parameter 'orders_filter' on template 'orders_filter_count'", + id="restricted_template_attribute_api", + ), + pytest.param( + "result = pydough.call_template('orders_revenue_by', labels={'arg_year': 'Month', 'arg_dimension': 'Year 1992'})", + None, + "The label 'Month' is not available for parameter 'arg_year' on template 'orders_revenue_by'", + id="restricted_template_parameter_api", + ), + ], +) +def test_pipeline_e2e_tpch_templates_errors( + pydough_impl: str, + columns: dict[str, str] | list[str] | None, + error_message: str, + get_sample_graph: graph_fetcher, + sqlite_tpch_db_context: DatabaseContext, +): + """ + Tests running bad PyDough code through the entire pipeline to verify that + a certain error is raised. + """ + graph: GraphMetadata = get_sample_graph("TPCH") + run_e2e_error_test( + pydough_impl, + error_message, + graph, + columns=columns, + database=sqlite_tpch_db_context, + ) + + @pytest.mark.parametrize( "graph_name, error_message", [ + # Attr with invalid name + pytest.param( + "INVALID_ATTRIBUTE_NAME", + "metadata for template attribute within graph 'INVALID_ATTRIBUTE_NAME' must be a JSON object containing a field 'name' and field 'name' must be a string", + id="invalid_attribute_name", + ), + # Dupplicated attribute name + pytest.param( + "DUPPLICATED_ATTRIBUTE_NAME", + "Already added template attribute 'attr1' in graph 'DUPPLICATED_ATTRIBUTE_NAME'", + id="duplicated_attribute_name", + ), + # Attr with invalid usage + pytest.param( + "INVALID_ATTRIBUTE_USAGE", + "template attribute 'attr1' in graph 'INVALID_ATTRIBUTE_USAGE' must be a dictionary where each key must be a string and each value must be a list where each element must be a string", + id="invalid_attribute_usage", + ), + # Attr with invalid type + pytest.param( + "INVALID_ATTRIBUTE_TYPE", + re.escape( + "Invalid type 'invalid_type' for attribute 'attr1' in graph 'INVALID_ATTRIBUTE_TYPE'. Must be one of: ['dict', 'float', 'int', 'list', 'pydough', 'str']" + ), + id="invalid_attribute_type", + ), + # Attribute with no options + pytest.param( + "NO_ATTRIBUTE_OPTIONS", + "Template attribute 'attr1' in graph 'NO_ATTRIBUTE_OPTIONS' must have at least one option defined.", + id="missing_options", + ), + # Attribute with invalid option label + pytest.param( + "INVALID_ATTRIBUTE_OPTION_LABEL", + "Option in attribute 'attr1' in graph 'INVALID_ATTRIBUTE_OPTION_LABEL' must be a JSON object containing a field 'label' and field 'label' must be a string", + id="invalid_option_label", + ), + # Attribute with invalid option value + pytest.param( + "INVALID_ATTRIBUTE_OPTION_VALUE", + re.escape( + "Option in attribute 'attr1' in graph 'INVALID_ATTRIBUTE_OPTION_VALUE' 'value' fields must be either all strings or all integers (not a mix, and no other type)." + ), + id="invalid_option_value", + ), + # Duplicated option label + pytest.param( + "DUPPLICATED_ATTRIBUTE_OPTION_LABEL", + "Duplicate option label: 'LABEL 1' for attribute 'attr1'. The label is already in use by attribute 'attr1' in graph 'DUPPLICATED_ATTRIBUTE_OPTION_LABEL'.", + id="duplicated_option_label", + ), + # Duplicates label option across attributes + pytest.param( + "DUPPLICATED_ATTRIBUTE_OPTION_LABEL_2", + "Duplicate option label: 'LABEL 1' for attribute 'attr2'. The label is already in use by attribute 'attr1' in graph 'DUPPLICATED_ATTRIBUTE_OPTION_LABEL_2'.", + id="duplicated_option_label_across", + ), + # Mixed option value type + pytest.param( + "MIXED_TYPE_ATTRIBUTE_OPTIONS", + re.escape( + "Option in attribute 'attr1' in graph 'MIXED_TYPE_ATTRIBUTE_OPTIONS' 'value' fields must be either all strings or all integers (not a mix, and no other type)." + ), + id="mixed_type_options", + ), # Attr with no definitions pytest.param( "NO_TEMPLATES_DEFINITIONS", "graph 'NO_TEMPLATES_DEFINITIONS' must be a JSON object containing a field 'definitions' and field 'definitions' must be a JSON array", id="missing_definitions", ), - # Attr with no options + # Template with invalid name pytest.param( - "NO_ATTIBUTE_OPTIONS", - "graph 'NO_ATTIBUTE_OPTIONS' must be a JSON object containing a field 'options' and field 'options' must be a JSON array", - id="missing_options", + "TEMPLATE_INVALID_NAME", + "Template definition 'name with space' in graph 'TEMPLATE_INVALID_NAME' must be a string that is a valid Python identifier", + id="invalid_template_name", + ), + # Template with invalid name reserved python word + pytest.param( + "TEMPLATE_NAME_PYTHON_RESERVED_WORD", + "Template definition 'continue' in graph 'TEMPLATE_NAME_PYTHON_RESERVED_WORD' must be a string that is not a Python reserved word or built-in name", + id="invalid_template_name_python_word", + ), + # Template with invalid name reserved pydough word + pytest.param( + "TEMPLATE_NAME_PYDOUGH_RESERVED_WORD", + "Template definition 'CALCULATE' in graph 'TEMPLATE_NAME_PYDOUGH_RESERVED_WORD' must be a string that is not a PyDough reserved word", + id="invalid_template_name_pydough_word", + ), + # Duplicated template name + pytest.param( + "TEMPLATE_NAME_DUPLICATED", + "Already added 'template_1' to graph 'TEMPLATE_NAME_DUPLICATED'", + id="invalid_template_name_duplicated", + ), + # Template with invalid paramter name + pytest.param( + "TEMPLATE_INVALID_PARAMETER_NAME", + re.escape( + "Parameter 'invalid_#@param' in template 'template_1' in graph 'TEMPLATE_INVALID_PARAMETER_NAME' must be a string that is a valid Python identifier" + ), + id="invalid_template_param_name", + ), + # Template with invalid parameter type + pytest.param( + "TEMPLATE_INVALID_PARAMETER_TYPE", + re.escape( + "Invalid type 'invalid_type' for the parameter 'parameter_1' of template 'template_1' in graph 'TEMPLATE_INVALID_PARAMETER_TYPE'. Must be one of: ['dict', 'float', 'int', 'list', 'pydough', 'str']" + ), + id="invalid_template_param_type", + ), + # No descrition on param + pytest.param( + "TEMPLATE_PARAM_NO_DESC", + re.escape( + "All parameters must have description in 'template_1' must be a JSON object containing a field 'description' and field 'description' must be a string" + ), + id="invalid_template_param_no_desc", + ), + # Invalid answer variable + pytest.param( + "TEMPLATE_INVALID_ANSWER_VAR", + re.escape( + "Answer variable '(no_valid_identifier)' of template 'template_1' in graph 'TEMPLATE_INVALID_ANSWER_VAR' must be a string that is a valid Python identifier" + ), + id="invalid_template_answer_var", + ), + pytest.param( + "TEMPLATE_INVALID_SOURCE_CODE", + re.escape( + "Template definition 'template_1' does not contain valid Python code: invalid syntax (, line 2)" + ), + id="invalid_template_source", + ), + pytest.param( + "TEMPLATE_INVALID_SOURCE_CODE_2", + re.escape( + "Internal error: failed to compile transformed template for 'template_1': no binding for nonlocal 'foo' found (, line 7)" + ), + id="invalid_template_source_2", ), ], ) From 169faaf7a40b1163ba86d2a1862c56de79ee4a35 Mon Sep 17 00:00:00 2001 From: john-sanchez31 Date: Mon, 7 Sep 2026 12:50:50 -0600 Subject: [PATCH 07/14] fixing metadata tests --- tests/test_metadata_errors.py | 2 +- tests/test_templates.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_metadata_errors.py b/tests/test_metadata_errors.py index 0311c45a5..4aed6d14b 100644 --- a/tests/test_metadata_errors.py +++ b/tests/test_metadata_errors.py @@ -79,7 +79,7 @@ def test_missing_property(get_sample_graph: graph_fetcher) -> None: pytest.param( "EXTRA_GRAPH_FIELDS", re.escape( - "graph 'EXTRA_GRAPH_FIELDS' must be a JSON object containing no fields except for ['additional definitions', 'collections', 'extra semantic info', 'functions', 'name', 'relationships', 'verified pydough analysis', 'version']" + "graph 'EXTRA_GRAPH_FIELDS' must be a JSON object containing no fields except for ['additional definitions', 'attributes', 'collections', 'extra semantic info', 'functions', 'name', 'relationships', 'templates', 'verified pydough analysis', 'version']" ), id="EXTRA_GRAPH_FIELDS", ), diff --git a/tests/test_templates.py b/tests/test_templates.py index f469a0579..170e5d1db 100644 --- a/tests/test_templates.py +++ b/tests/test_templates.py @@ -509,7 +509,7 @@ def test_pipeline_e2e_tpch_templates_errors( pytest.param( "INVALID_ATTRIBUTE_TYPE", re.escape( - "Invalid type 'invalid_type' for attribute 'attr1' in graph 'INVALID_ATTRIBUTE_TYPE'. Must be one of: ['dict', 'float', 'int', 'list', 'pydough', 'str']" + "Invalid type 'invalid_type' for attribute 'attr1' in graph 'INVALID_ATTRIBUTE_TYPE'. Must be one of: ['datetime', 'dict', 'float', 'int', 'list', 'pd.DataFrame', 'pydough', 'str']" ), id="invalid_attribute_type", ), @@ -595,7 +595,7 @@ def test_pipeline_e2e_tpch_templates_errors( pytest.param( "TEMPLATE_INVALID_PARAMETER_TYPE", re.escape( - "Invalid type 'invalid_type' for the parameter 'parameter_1' of template 'template_1' in graph 'TEMPLATE_INVALID_PARAMETER_TYPE'. Must be one of: ['dict', 'float', 'int', 'list', 'pydough', 'str']" + "Invalid type 'invalid_type' for the parameter 'parameter_1' of template 'template_1' in graph 'TEMPLATE_INVALID_PARAMETER_TYPE'. Must be one of: ['datetime', 'dict', 'float', 'int', 'list', 'pd.DataFrame', 'pydough', 'str']" ), id="invalid_template_param_type", ), From 68734763768f649c228de7e13429f85a9f84fcba Mon Sep 17 00:00:00 2001 From: john-sanchez31 Date: Tue, 8 Sep 2026 12:27:17 -0600 Subject: [PATCH 08/14] adding documentation [run ci][run dialects] --- documentation/dsl.md | 253 +++++++++++++++++ documentation/metadata.md | 274 +++++++++++++++++++ documentation/usage.md | 67 +++++ pydough/unqualified/unqualified_transform.py | 18 +- 4 files changed, 608 insertions(+), 4 deletions(-) diff --git a/documentation/dsl.md b/documentation/dsl.md index f269ba404..0dbad3e44 100644 --- a/documentation/dsl.md +++ b/documentation/dsl.md @@ -23,6 +23,7 @@ This page describes the specification of the PyDough DSL. The specification incl * [range_collection](#range_collection) * [dataframe_collection](#dataframe_collection) * [View/Table Collections (via to_table)](#view_collection) +- [PyDough Templates](#templates) - [Larger Examples](#larger-examples) * [Example 1: Highest Residency Density States](#example-1-highest-residency-density-states) * [Example 2: Yearly Trans-Coastal Shipments](#example-2-yearly-trans-coastal-shipments) @@ -1776,6 +1777,258 @@ Result: 2 12 1 ``` + +## PyDough Templates + +Once a template is registered in metadata (see [Templates](metadata#templates) for how to define one), it can be used in two ways: + +- **Directly**, as an ordinary PyDough function call, passing raw literal/PyDough values as arguments. +- **Through the API**, via [`pydough.call_template`](usage#call_template_api), passing human-facing labels instead of raw values. + +Both approaches ultimately call the same generated function and produce the same result — the only difference is how the arguments are supplied. Regardless of which is used, the result of a template call can be treated like any other PyDough value: chained into further PyDough operations, passed into another template, or (if the template's `answer_variable` is a plain literal) used directly in Python. + + +### Calling a Template Directly + +A loaded template behaves like any other PyDough function — call it by name, passing values positionally in the order its `parameters` were defined (see [Parameter Numeration](#template-parameter-numeration)). + +Example: using the `orders_filter_count` template directly, with a raw PyDough filter condition, to find the top 5 customers by number of orders placed in 1996 above a price of 3000: + +Template definition: +```json +{ + "name": "orders_filter_count", + "description": "Counts the number of orders placed that satisfied the given condition.", + "parameters": { + "orders_filter": { + "type": "pydough", + "description": "Condition(s) for the order to make it count" + } + }, + "source": "result = COUNT(orders.WHERE({1}))", + "answer_variable": "result" +} +``` + +```python +customers.CALCULATE( + key, + n_orders=orders_filter_count( + (total_price > 3000) & (YEAR(order_date) == 1996) + ), +).TOP_K(5, by=(n_orders.DESC(), key.ASC())) +``` + + +### Calling a Template via `pydough.call_template` + +Instead of passing raw values, `pydough.call_template` looks up each argument by a human-facing **label**, resolving it to the underlying value through the graph's [attributes](metadata#template-attributes). See [`pydough.call_template`](usage#call_template_api) for the full parameter/error reference. + +Example: using the `orders_filter_count` template via the API, with the label `"High priority"` resolving to a PyDough filter condition: + +```python +result = customers.CALCULATE( + key, + n_orders=pydough.call_template( + "orders_filter_count", labels={"orders_filter": "High priority"} + ), +).TOP_K(5, by=(n_orders.DESC(), key.ASC())) + +pydough.to_df(result) +``` + +Because `call_template` returns an ordinary `UnqualifiedNode` (or literal), its result can be used exactly like any other PyDough value — including inside a filter, as shown here to find the nations with the most customers who have placed an order above 3000: + +```python +selected_orders = pydough.call_template( + "orders_filter_count", labels={"orders_filter": "Price above 3000"} +) +result = nations.CALCULATE( + name, n_customers=COUNT(customers.WHERE(selected_orders > 0)) +).TOP_K(3, by=(n_customers.DESC(), name.ASC())) + +pydough.to_df(result) +``` + + +### Chaining Templates Together + +The result of one template call can be fed directly into another template as an argument, letting templates build on each other without needing to re-express the earlier result in raw PyDough. Works the same way whether the first call was made directly or through the API. + +Example: calling `orders_revenue_by` via the API to partition order revenue for 1997 by month, then passing that result into `top_bottom_comparison` to find the highest- and lowest-performing months: + +Template definitions: +```json +{ + "name": "orders_revenue_by", + "description": "Calculates the revenue of orders in a given year, partitioned by a specified dimension.", + "parameters": { + "arg_year": { + "type": "int", + "description": "The year for which to calculate the revenue of orders." + }, + "arg_dimension": { + "type": "pydough", + "description": "The dimension by which to partition the revenue calculation." + } + }, + "source": "result = orders.WHERE(({1} == YEAR(order_date))).CALCULATE(revenue=order_revenue(), dimension=({2})).PARTITION(name=\"orders_groups\", by=dimension).CALCULATE(dimension, segment_revenue=SUM(orders.revenue))\n", + "answer_variable": "result" +}, +{ + "name": "top_bottom_comparison", + "description": "Compares the top and bottom groups of a partitioned orders", + "parameters": { + "arg_partitioned_orders": { + "type": "pydough", + "description": "The partitioned orders to compare." + }, + "arg_calculation": { + "type": "pydough", + "description": "The metric by which to compare the groups." + } + }, + "source": "result = {1}.CALCULATE(dimension, segment_revenue, comparison_value={2}).WHERE(ABSENT(PREV(segment_revenue, by=segment_revenue.DESC())) | ABSENT(NEXT(segment_revenue, by=segment_revenue.DESC())) )", + "answer_variable": "result" +} +``` + +```python +orders_segmentation = pydough.call_template( + "orders_revenue_by", labels={"arg_year": "Year 1997", "arg_dimension": "Month"} +) + +final_result = top_bottom_comparison(orders_segmentation, AVG(orders.revenue)) + +pydough.to_df(final_result) +``` + + +### Templates Returning Literals + +Not every template needs to return a PyDough collection/expression — a template's `answer_variable` can just as easily hold a plain literal (see [Definition Field: `source`](metadata#template-source)). Its result can then be used as an ordinary Python/PyDough value, such as a threshold in a later filter. + +Example: calling `multiply_by_2` via the API to compute a minimum account balance from a label, then using that value directly in a `WHERE`: + +Template definition: +```json +{ + "name": "multiply_by_2", + "description": "Receives an integer and return its multiplication by 2.", + "parameters": { + "base_number": { + "type": "int", + "description": "Number being multiply by 2" + } + }, + "source": "result = {1} * 2\n", + "answer_variable": "result" +} +``` + +```python +min_account_balance = pydough.call_template( + "multiply_by_2", labels={"base_number": "Year 1992"} +) + +selected_customers = customers.WHERE(account_balance >= min_account_balance) +final_result = TPCH.CALCULATE(n_custs=COUNT(selected_customers)) + +pydough.to_df(final_result) +``` + + +### Templates Producing a User-Generated Collection + +A template's `source` can also produce a **user-generated collection** — e.g. one built from an in-memory `pd.DataFrame` via `pydough.dataframe_collection` — rather than deriving from the graph itself. This works like any other template, but the `pd.DataFrame` and `list` typed parameters let external, ad hoc data be wrapped into a proper collection and then combined with the rest of the graph (e.g. via `CROSS`, `WHERE`, `CALCULATE`) just as if it were a regular collection. + +Example template definition, wrapping a `pd.DataFrame` into a named dataframe collection with a list of unique columns: + +```json +{ + "name": "dataframe_input_collection", + "description": "Generates a dataframe collection from the given parameters", + "parameters": { + "collection_name": { + "type": "str", + "description": "Name for the generated dataframe collection" + }, + "new_df": { + "type": "pd.DataFrame", + "description": "New dataframe to create the collection with" + }, + "unique_columns": { + "type": "list", + "description": "List of unique column names for the dataframe collection" + } + }, + "source": "result = pydough.dataframe_collection({1}, {2}, {3})", + "answer_variable": "result" +} +``` + +Example usage: calling `dataframe_input_collection` directly with an input DataFrame, then crossing the resulting collection with a filtered set of orders: + +```python +input_df = pd.DataFrame( + { + "cust_id": [1, 2, 3], + "customer_name": ["customer_1", "customer_2", "customer_3"], + } +) + +df_collection = dataframe_input_collection( + "customers_collection", input_df, ["cust_id"] +) + +selected_orders = orders.WHERE(ISIN(key, (1, 2, 3))).CALCULATE(key, clerk) + +( + df_collection.CALCULATE(cust_id, customer_name) + .CROSS(selected_orders) + .WHERE(cust_id == key) + .CALCULATE(cust_id, customer_name, key, clerk) +) +``` + + +### Recursive Templates + +A template's `source` can call itself, since it is compiled into a regular Python function (see [Definition Field: `source`](metadata#template-source)). Recursive templates are called the same way as any other template — directly or via the API — with no special handling needed at the call site. + +Example: calling the recursive `cumulative_orders_counter` template directly, with different `base_year`/`last_year` ranges: + +Template definition: +```json +{ + "name": "cumulative_orders_counter", + "description": "Calculates the cumulative counter of all orders placed from a base year through a given last year. Recursively sums the current year's counter and the cumulative counter of all next years.", + "parameters": { + "base_year": { + "type": "int", + "description": "The year through which to calculate cumulative revenue." + }, + "last_year": { + "type": "int", + "description": "The earliest year to include in the cumulation (recursion base case)." + } + }, + "source": "assert base_year <= last_year\nif {1} == {2}:\n result = COUNT(orders.WHERE(YEAR(order_date) == base_year))\nelse:\n result = (\n COUNT(orders.WHERE(YEAR(order_date) == base_year)) + cumulative_orders_counter(base_year + 1, last_year))\n", + "answer_variable": "result" +} +``` + +```python +result = TPCH.CALCULATE( + y_1994=cumulative_orders_counter(1994, 1994), + y_1994_1996=cumulative_orders_counter(1994, 1996), + y_1994_1998=cumulative_orders_counter(1994, 1998), +) + +pydough.to_df(result) +``` + + ## Larger Examples diff --git a/documentation/metadata.md b/documentation/metadata.md index e6230bf5b..f6d06e637 100644 --- a/documentation/metadata.md +++ b/documentation/metadata.md @@ -26,6 +26,7 @@ This page document the exact format that the JSON files containing PyDough metad - [Function Deducers](#function-deducers) * [Function Deducer Type: Constant](#function-deducer-type-constant) * [Function Deducer Type: Select Argument](#function-deducer-type-select-argument) +- [Templates](#templates) - [PyDough Type Strings](#pydough-type-strings) - [Metadata Examples](#metadata-examples) * [Example: TPC-H](#example-tpch) @@ -493,6 +494,279 @@ Below are several examples the JSON for such deducers: - Returns the type of the first argument: `{"type": "select argument", "value": 0}` - Returns the type of the second argument: `{"type": "select argument", "value": 1}` + + +## Templates + +A template registers a reusable, **parametrized function** in metadata, which becomes available for later use either as a normal PyDough function call or through the `pydough.call_template()` API using human-readable **labels** instead of raw values. More about templates' usage [here](dsl#pydough-templates) + +Despite the name, a template is not limited to a single PyDough expression — its `source` is effectively the **body of a Python function**. It can contain PyDough collection/expression code, arbitrary Python control flow (`if`/`else`, `assert`, loops), plain Python/PyDough literals (`int`, `str`, `dict`, …), and even recursive calls back into itself. See [Definition Field: `source`](#template-source) for details and an example. + +All template-related metadata lives under a single top-level `"templates"` key, with two sub-parts: + +```json +"templates": { + "attributes": [ ... ], + "definitions": [ ... ] +} +``` + +- **`attributes`** — (optional) the shared pool of *values* (with human-readable labels) that templates can use as an argument. Only needed if a template will be invoked via `call_template()`; a template that is only ever called directly with literal arguments does not require any attributes. +- **`definitions`** — the templates themselves: the PyDough/Python source, its parameters, and a brief description. + +Both attribute values and template parameters are typed using a shared, fixed set of [supported types](#template-supported-types); an attribute can only supply a value to a parameter if their types match. + + +### Attributes + +An attribute describes a reusable, labeled set of values that one or more templates can use as an argument. Attributes are exclusively used with `call_template()` — they map a user-facing **label** (e.g. `"Year 1996"`) to the actual PyDough/Python **value** (e.g. `1996`) that gets substituted into the template. + +Every JSON object describing an attribute has the following fields: +- `name` (required): unique identifier for the attribute. Referenced by templates that use it. +- `usage` (required): which templates (and optionally which parameters of those templates) may use this attribute. An empty list `[]` means "usable by all templates." [See here](#attribute-usage-restriction) for the specification of this field. +- `type` (required): the value's type. This is used to verify if the value match the template's parameter. To know more about the supported types [See here](#template-supported-types) +*Note*: `pydough` is a special type that represents a raw PyDough expression/snipet. +- `description` (required): a semantic description of what the attribute represents. +- `options` (required): the set of valid `{label, value}` pairs for this attribute. [See here](#attribute-options) for the specification of these objects. + + +#### Attribute Field: `usage` + +The `usage` field must unambiguously express **which templates**, and **which specific parameter(s) of those templates**, can use the attribute. This field is a JSON object (map) keyed by template name, whose value is a list of parameter names within that template allowed to use the attribute: + +```json +"usage": { + "orders_revenue_by": ["arg_dimension"], + "top_bottom_comparison": ["arg_calculation"] +} +``` + +An empty object `{}` means the attribute is usable by any parameter of any template. Also an empty list `[]` for a template indicates that the attribute can be used in any parameter of the template. + + +#### Attribute Field: `options` + +Each entry in `options` is a JSON object with the following fields: +- `label` (required): the human-facing name shown to / provided by the user (e.g. in a question or UI). This is what `call_template()` accepts. This label must be unique among all labels in the graph. +- `value` (required): the literal or PyDough snippet substituted into the template source when this label is selected. + + +##### Options with `list` or `dict` Values + +For attributes whose `type` is `list` or `dict`, `value` cannot be written as a native JSON array/object, because a raw JSON array or object cannot hold an unquoted PyDough expression (e.g. `pydough.expression`) — JSON only allows quoted strings, numbers, booleans, `null`, or nested arrays/objects as elements. Instead, the entire list or dict literal is written as a single string, using Python/PyDough syntax. This string is substituted verbatim into the source (it is not JSON-decoded). + +Example of the structure of the metadata for an attribute named `years`: + +```json +{ + "name": "years", + "usage": {"orders_revenue_by": ["arg_year"], "multiply_by_2": ["base_number"]}, + "type": "int", + "description": "The year for which to calculate the revenue of orders.", + "options": [ + { "label": "Year 1992", "value": 1992}, + { "label": "Year 1993", "value": 1993} + ] +} +``` + +Example of the structure of the metadata for an attribute of type `pydough`, whose options are filter conditions: + +```json +{ + "name": "order_filter_condition", + "usage": { "orders_filter_count": ["orders_filter"] }, + "type": "pydough", + "description": "Condition(s) by which orders can be filtered", + "options": [ + { "label": "Status O", "value": "(order_status == 'O')" }, + { "label": "Price above 3000", "value": "(total_price > 3000)" }, + { "label": "March", "value": "(MONTH(order_date) == 3)" }, + { "label": "High priority", "value": "ISIN(order_priority, ('1-URGENT', '2-HIGH'))" } + ] +} +``` + +Example of the structure of the metadata for an attribute of type `str`: + +```json +{ + "name": "order_priority_levels", + "usage": { "order_lvl_priority": ["level"] }, + "type": "str", + "description": "Level of priority for an order", + "options": [ + { "label": "LEVEL 0", "value": "other" }, + { "label": "LEVEL 1", "value": "low" }, + { "label": "LEVEL 2", "value": "medium" }, + { "label": "LEVEL 3", "value": "high" } + ] +} +``` + +Example of the structure of the metadata for an attribute of type `list`: + +```json +{ + "name": "dataframe_collection_column", + "usage": { "generate_df_collection": ["col1"] }, + "type": "list", + "description": "List with various types of colors to be used in the dataframe collection", + "options": [ + { "label": "COLORS", "value": "['blue', 'red', 'yellow', 'purple']" } + ] +} +``` + +An analogous `dict`-typed attribute would follow the same pattern: + +```json +{ "label": "DICT VALUE", "value": "{'key': pydough.expression}" } +``` + + +### Template Definitions + +A definition describes one callable template: its parameters, its PyDough source code, and what it returns. + +Every JSON object describing a template definition has the following fields: +- `name` (required): unique template name, which must be a valid PyDough identifier and must not overlap with the name of other collections/properties/relationships/functions within the graph. Becomes both the generated PyDough function name and the string passed to `call_template()`. This name will become reserved like other function names in PyDough (`COUNT`, `LOWER`, etc.) +- `description` (required): a semantic description of what the template computes. +- `parameters` (required, `{}` if none): the template's arguments, keyed by parameter name. [See here](#template-parameters) for the specification of these objects. +- `source` (required): the PyDough code body. Must assign to the variable named in `answer_variable`. Positional placeholders (`{1}`, `{2}`, …) are substituted with parameter values/expressions before evaluation. All details about `source` for template definitions can be found [here](#template-source). +- `answer_variable` (required): name of the variable inside `source` whose value the generated function returns. + + +#### Definition Field: `parameters` + +`parameters` is a JSON object (map) keyed by parameter name. Each value is a JSON object with the following fields: +- `type` (required): `"int"`, `"pydough"`, etc. `"pydough"` signals the value is a raw expression rather than a literal. To know more about the supported types [See here](#template-supported-types) +- `description` (required): a semantic description of the parameter's purpose. + + +##### Parameter Numeration + +Parameters are numbered implicitly by their position within the `parameters` object: the first key defined is parameter `1`, the second is parameter `2`, and so on. This numeration determines which placeholder each parameter fills in `source` — parameter `1` replaces every `{1}` in the source code, parameter `2` replaces every `{2}`, etc. + +> **Note:** this relies on the `parameters` object preserving key insertion order. All standard JSON parsers used by this project (e.g. Python's `json` module) preserve insertion order, but this ordering is not part of the JSON specification itself — reordering, alphabetizing, or otherwise regenerating the keys of a `parameters` object will change which placeholder each parameter maps to. + +Example of the structure of the metadata for a parameter map with two parameters — `arg_year` is `{1}` and `arg_dimension` is `{2}` in `source`: + +```json +"parameters": { + "arg_year": { + "type": "int", + "description": "The year for which to calculate the revenue of orders." + }, + "arg_dimension": { + "type": "pydough", + "description": "The dimension by which to partition the revenue calculation." + } +} +``` + +Example of the structure of the metadata for a template named `orders_revenue_by` (depends on another template, `order_revenue`): + +```json +{ + "name": "orders_filter_count", + "description": "Counts the number of orders placed that satisfied the given condition.", + "parameters": { + "orders_filter": { + "type": "pydough", + "description": "Condition(s) for the order to make it count" + } + }, + "source": "result = COUNT(orders.WHERE({1}))", + "answer_variable": "result" +} +``` + +Example of the structure of the metadata template named `order_revenue`: + +```json +{ + "name": "order_revenue", + "dependencies": [], + "description": "Calculates the revenue of an order, which is the sum of the extended price * (1 - discount) for all line items in the order.", + "parameters": {}, + "source": "result = SUM(lines.extended_price * (1 - lines.discount))\n", + "answer_variable": "result" +} +``` + + +### Naming Collisions + +`name` (for both attributes and template definitions) must be validated against: +1. Other attribute/template names (no duplicates within their own graph). +2. Existing PyDough built-in functions (e.g. `GETPART`, `STRCOUNT`) — a template cannot shadow one of these. +3. Python builtins, to the extent the generated function is exposed in a namespace where that matters. + + + +### Supported Types + +Both the `type` field of an [attribute](#template-attributes) and the `type` field of a [template parameter](#template-parameters) draw from the same fixed set of supported types. When an attribute is used to supply a value for a parameter, their `type`s must match. + +The currently supported types are: + +| Type | Description | +|---|---| +| `str` | A string literal. | +| `int` | An integer literal. | +| `float` | A floating-point literal. | +| `list` | A Python list literal. Written as a [string](#attribute-options-list-dict) in attribute options. | +| `dict` | A Python dict literal. Written as a [string](#attribute-options-list-dict) in attribute options. | +| `pydough` | A raw, untyped PyDough expression/snippet (e.g. a column reference, a `CALCULATE` call, or the output of another template). | +| `pd.DataFrame` | A pandas DataFrame. | +| `datetime` | A `datetime` value. | + +For template parameters, every type except `pydough` is written as a type annotation on the generated function's signature (e.g. a parameter of type `int` named `base_year` becomes `base_year: int`). Parameters of type `pydough` are left unannotated, since they represent arbitrary PyDough expressions rather than a single Python type. + + +### Definition Field: `source` + +`source` is not limited to a single PyDough expression — it is the **body of a Python function**, and can contain anything valid inside one: `if`/`else` branching, `assert` statements, intermediate variables, loops, and even recursive calls back to the template itself (referenced by its own `name`). The only requirements are: + +- The code must assign to the variable named in `answer_variable` along every code path. +- Positional placeholders (`{1}`, `{2}`, …) are substituted with the corresponding parameter names — following the [parameter numeration](#template-parameter-numeration) — before the code is parsed. + +Because of this, a template's `answer_variable` does not need to hold a PyDough collection/expression — it can just as easily be a plain Python/PyDough literal (`int`, `str`, `dict`, etc.), the result of ordinary Python control flow, or the result of the template recursively calling itself. + +Example of the structure of the metadata for a recursive template, which sums a per-year order counter from `base_year` through `last_year`: + +```json +{ + "name": "cumulative_orders_counter", + "description": "Calculates the cumulative counter of all orders placed from a base year through a given last year. Recursively sums the current year's counter and the cumulative counter of all next years.", + "parameters": { + "base_year": { "type": "int", "description": "The year through which to calculate cumulative revenue." }, + "last_year": { "type": "int", "description": "The earliest year to include in the cumulation (recursion base case)." } + }, + "source": "assert base_year <= last_year\nif {1} == {2}:\n result = COUNT(orders.WHERE(YEAR(order_date) == base_year))\nelse:\n result = (\n COUNT(orders.WHERE(YEAR(order_date) == base_year)) + cumulative_orders_counter(base_year + 1, last_year))\n", + "answer_variable": "result" +} +``` + +This is equivalent to generating the following function: + +```python +def cumulative_orders_counter(base_year: int, last_year: int): + assert base_year <= last_year + if base_year == last_year: + result = COUNT(orders.WHERE(YEAR(order_date) == base_year)) + else: + result = ( + COUNT(orders.WHERE(YEAR(order_date) == base_year)) + + cumulative_orders_counter(base_year + 1, last_year) + ) + return result +``` + +TODO: Add link to the part how the templates are used + + ## PyDough Type Strings diff --git a/documentation/usage.md b/documentation/usage.md index 8997cc678..70cfcddb3 100644 --- a/documentation/usage.md +++ b/documentation/usage.md @@ -16,6 +16,7 @@ This document describes how to set up & interact with PyDough. For instructions * [`pydough.to_table`](#pydoughto_table) - [Transformation APIs](#transformation-apis) * [`pydough.from_string`](#pydoughfrom_string) + * [`pydough.call_template`](#call_template_api) - [Exploration APIs](#exploration-apis) * [`pydough.explain_structure`](#pydoughexplain_structure) * [`pydough.explain`](#pydoughexplain) @@ -966,6 +967,72 @@ ORDER BY 3 DESC ``` + +### `pydough.call_template` + +The `call_template` API invokes a previously registered [template](metadata#templates) using human-facing **labels** instead of raw PyDough/Python values. Each label is resolved against the `options` of the graph's [attributes](metadata#template-attributes) to find the concrete value (and its type) to pass into the template, after which the template is called like a normal PyDough function and the result is returned. + +#### Syntax +```python +def call_template( + name: str, + labels: dict[str, str], +) -> Any: +``` + +- `name`: the name of the template to call, as registered in `pydough.active_session.metadata.templates_definitions`. +- `labels`: a mapping from **template parameter name** to a **label string** (e.g. `{"arg_year": "Year 1998", "arg_dimension": "Customer Market Segment"}`). Each label must appear in the `options` of exactly one attribute in the active session's graph, and that attribute's `type` must match the corresponding template parameter's `type`. + +#### Return value + +The return type is `Any`, not necessarily an `UnqualifiedNode`. Because a template's `source` is a full Python function body (see [Definition Field: `source`](metadata#template-source)), the result of calling it can be an `UnqualifiedNode` (on which `explain()`, `to_sql()`, or `to_df()` can be called), or it can be a plain literal (`int`, `str`, `dict`, etc.) if that's what the template's `answer_variable` ultimately holds. + +#### Resolution process + +For each `(arg_name, label)` pair in `labels`, `call_template` does the following: + +1. Confirms `arg_name` is a real parameter of the named template; if not, raises an error listing the template's actual parameter names as suggestions. +2. Searches every attribute registered in the active session's graph for one whose `options` contains `label`. +3. If a match is found, verifies that the matching attribute is allowed to supply this template/parameter combination, based on the attribute's [`usage`](metadata#attribute-usage-restriction) field. +4. Verifies that the matching attribute's `type` matches the template parameter's `type`. +5. Resolves `label` to its corresponding `value`, to be substituted into the generated call. + +Once every entry in `labels` has been resolved, the template is invoked with the resolved arguments (as a normal PyDough/Python call), and the result is returned exactly as produced by the template — no additional check is made that it's an `UnqualifiedNode`. + +#### Errors + +`call_template` raises `ValueError` in the following cases: + +| Condition | Message behavior | +|---|---| +| No metadata is loaded in the active session | `"No metadata loaded in the current active session"` | +| `name` doesn't match any registered template | Reports the template isn't found; if similarly-named templates exist, suggests them. | +| The graph has no attributes registered at all | Reports that no attributes are available for the template. | +| `arg_name` in `labels` isn't a parameter of the template | Reports the parameter doesn't exist; suggests the template's actual parameter names. | +| `label` isn't restricted to this template/parameter by its attribute's `usage` | Reports the label isn't available for that parameter on that template. | +| The matching attribute's `type` differs from the parameter's expected `type` | Reports the type mismatch between the attribute and the parameter. | +| `label` doesn't appear in any attribute's `options` | Reports the label wasn't found in any attribute's options. | +| A required template parameter has no corresponding entry in `labels` | Reports the template is missing labels for those parameter(s), by name. | + +#### Example + +Using the `orders_revenue_by` template and the `years` / `order_dimensions` attributes shown [here](metadata#templates): + +```python +import pydough + +graph = pydough.active_session.load_metadata_graph("demos/metadata/tpch_demo_graph.json", "TPCH") +pydough.active_session.connect_database("sqlite", database="tpch.db") + +result = pydough.call_template( + "orders_revenue_by", + labels={"arg_year": "Year 1996", "arg_dimension": "Customer Region"}, +) +pydough.to_df(result) +``` + +This resolves `"Year 1996"` to `1996` via the `years` attribute and `"Customer Region"` to `customer.nation.region.name` via the `order_dimensions` attribute, then calls the generated `orders_revenue_by(1996, customer.nation.region.name)` function and returns its result. + ## Exploration APIs diff --git a/pydough/unqualified/unqualified_transform.py b/pydough/unqualified/unqualified_transform.py index 918b73ba8..b216da236 100644 --- a/pydough/unqualified/unqualified_transform.py +++ b/pydough/unqualified/unqualified_transform.py @@ -624,8 +624,6 @@ def call_template( one attribute in the active session's graph, and that attribute's type must match the corresponding template parameter's type. - TODO: What to do with repeated labels? Check before add the option? - Returns: The `UnqualifiedNode` produced by calling the template with the resolved arguments, ready for further chaining or execution @@ -669,6 +667,16 @@ def call_template( pydough.active_session.metadata.templates_definitions[name] ) + missing_params: list[str] = [ + p for p in calling_template.parameters if p not in labels + ] + if missing_params: + missing_params_str: str = ", ".join(missing_params) + raise ValueError( + f"Template {name!r} is missing a label for the following " + f"parameter(s): {missing_params_str}" + ) + available_attributes: dict[str, AttributeMetadata] = ( pydough.active_session.metadata.templates_attributes ) @@ -698,8 +706,10 @@ def call_template( # Check if the attribute is available for this template and argument if attribute.usage != {} and ( name not in attribute.usage - or attribute.usage[name] == [] # No argument restrictions - or arg_name not in attribute.usage[name] + or ( + attribute.usage[name] != [] + and arg_name not in attribute.usage[name] + ) ): raise ValueError( f"The label {label!r} is not available for parameter '{arg_name}' on template '{name}'" From 2b034606525ca4b0e5b5e5d927e088835cf17ff0 Mon Sep 17 00:00:00 2001 From: john-sanchez31 Date: Tue, 8 Sep 2026 12:45:30 -0600 Subject: [PATCH 09/14] fixing test [run ci][run dialects] --- tests/test_templates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_templates.py b/tests/test_templates.py index 170e5d1db..fafbff3df 100644 --- a/tests/test_templates.py +++ b/tests/test_templates.py @@ -440,7 +440,7 @@ def test_pipeline_e2e_tpch_templates( pytest.param( "result = pydough.call_template('orders_filter_count', labels={'no_param': 'LABEL 1'})", None, - "Template 'orders_filter_count' doesn't have a paramater called 'no_param'. Did you mean: orders_filter", + "Template 'orders_filter_count' is missing a label for the following parameter(s): orders_filter", id="wrong_template_arguments_api", ), pytest.param( From 2684c7bdbb42a5cd8901b494935c423248d9524c Mon Sep 17 00:00:00 2001 From: john-sanchez31 Date: Tue, 8 Sep 2026 15:30:14 -0600 Subject: [PATCH 10/14] fixing error test [run ci][run dialects] --- tests/test_templates.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_templates.py b/tests/test_templates.py index fafbff3df..acf1bb7da 100644 --- a/tests/test_templates.py +++ b/tests/test_templates.py @@ -440,7 +440,9 @@ def test_pipeline_e2e_tpch_templates( pytest.param( "result = pydough.call_template('orders_filter_count', labels={'no_param': 'LABEL 1'})", None, - "Template 'orders_filter_count' is missing a label for the following parameter(s): orders_filter", + re.escape( + "Template 'orders_filter_count' is missing a label for the following parameter(s): orders_filter" + ), id="wrong_template_arguments_api", ), pytest.param( From 87762003e5e4c85c80574be1bf093a9a0596583f Mon Sep 17 00:00:00 2001 From: john-sanchez31 Date: Wed, 9 Sep 2026 08:14:25 -0600 Subject: [PATCH 11/14] to_table test databricks fix [run databricks] --- tests/test_metadata/databricks_sample_graphs.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_metadata/databricks_sample_graphs.json b/tests/test_metadata/databricks_sample_graphs.json index 18b0462f7..3da5e771a 100644 --- a/tests/test_metadata/databricks_sample_graphs.json +++ b/tests/test_metadata/databricks_sample_graphs.json @@ -1083,7 +1083,7 @@ "description": "Region to filter the nations by" } }, - "source": "my_nations = nations.WHERE(region.name == {1})\nnations_tmp = pydough.to_table(my_nations, name='region_nations_t1', temp=False, replace=True)\nresult = nations_tmp.CALCULATE(name)", + "source": "my_nations = nations.WHERE(region.name == {1})\nnations_tmp = pydough.to_table(my_nations, name='region_nations_t1', write_path='e2e_tests_db.to_table_py313.region_nations_t1', temp=False, replace=True)\nresult = nations_tmp.CALCULATE(name)", "answer_variable": "result" }, { From f22077b252290b396cee9c7f479c56facf697009 Mon Sep 17 00:00:00 2001 From: john-sanchez31 Date: Wed, 9 Sep 2026 08:35:59 -0600 Subject: [PATCH 12/14] testing [run ci][run dialects] From e66d1b0d3541ce0254e15303c227def60ef75bfa Mon Sep 17 00:00:00 2001 From: john-sanchez31 Date: Wed, 9 Sep 2026 15:03:37 -0600 Subject: [PATCH 13/14] second try fix databricks [run databricks] --- tests/test_metadata/databricks_sample_graphs.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_metadata/databricks_sample_graphs.json b/tests/test_metadata/databricks_sample_graphs.json index 3da5e771a..3517d292a 100644 --- a/tests/test_metadata/databricks_sample_graphs.json +++ b/tests/test_metadata/databricks_sample_graphs.json @@ -1083,7 +1083,7 @@ "description": "Region to filter the nations by" } }, - "source": "my_nations = nations.WHERE(region.name == {1})\nnations_tmp = pydough.to_table(my_nations, name='region_nations_t1', write_path='e2e_tests_db.to_table_py313.region_nations_t1', temp=False, replace=True)\nresult = nations_tmp.CALCULATE(name)", + "source": "my_nations = nations.WHERE(region.name == {1})\nnations_tmp = pydough.to_table(my_nations, name='region_nations_t1', write_path='e2e_tests_db.to_table_py313.region_nations_t1', temp=True, replace=True)\nresult = nations_tmp.CALCULATE(name)", "answer_variable": "result" }, { From 1cb03d360f1fe66a2c7ce9ac841dfd23e557a99f Mon Sep 17 00:00:00 2001 From: john-sanchez31 Date: Wed, 9 Sep 2026 15:56:47 -0600 Subject: [PATCH 14/14] adding metadata documentation link [run ci][run dialects] --- documentation/metadata.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/metadata.md b/documentation/metadata.md index ff0c67960..024e444fa 100644 --- a/documentation/metadata.md +++ b/documentation/metadata.md @@ -776,7 +776,7 @@ def cumulative_orders_counter(base_year: int, last_year: int): return result ``` -TODO: Add link to the part how the templates are used +You can find how to use templates and more examples [here](dsl#pydough-templates) ## PyDough Type Strings