From 3420f450cb83354381441853a530c7988e69c8fb Mon Sep 17 00:00:00 2001 From: knassre-bodo Date: Wed, 13 May 2026 09:52:17 -0700 Subject: [PATCH 01/44] Start of explode tests, need to fill out rest --- tests/test_pipeline_tpch_custom.py | 82 ++++++++++++++++++++++++++++++ tests/testing_utilities.py | 22 ++++++++ 2 files changed, 104 insertions(+) diff --git a/tests/test_pipeline_tpch_custom.py b/tests/test_pipeline_tpch_custom.py index d97977032..b05632bc3 100644 --- a/tests/test_pipeline_tpch_custom.py +++ b/tests/test_pipeline_tpch_custom.py @@ -4071,6 +4071,88 @@ ), id="quantile_function_test_4", ), + pytest.param( + PyDoughPandasTest( + "result = regions.CALCULATE(region_name=name, nation_names=ARRAY_COLLECT(nations.name)).ORDER_BY(region_name)", + "TPCH", + lambda: pd.DataFrame( + { + "region_name": [ + "AFRICA", + "AMERICA", + "ASIA", + "EUROPE", + "MIDDLE EAST", + ], + "nation_names": [ + ["ALGERIA", "ETHIOPIA", "KENYA", "MOROCCO", "MOZAMBIQUE"], + ["ARGENTINA", "BRAZIL", "CANADA", "PERU", "UNITED STATES"], + ["CHINA", "INDIA", "INDONESIA", "JAPAN", "VIETNAM"], + [ + "FRANCE", + "GERMANY", + "ROMANIA", + "RUSSIA", + "UNITED KINGDOM", + ], + ["EGYPT", "IRAN", "IRAQ", "JORDAN", "SAUDI ARABIA"], + ], + } + ), + "array_data_01", + order_sensitive=True, + skipped_dialects={"ANSI", "SQLITE"}, + ), + id="array_data_01", + ), + pytest.param( + PyDoughPandasTest( + "array_data = regions.CALCULATE(nation_names=ARRAY_COLLECT(nations.name))\n" + "result = array_data.EXPLODE(nation_names, index_name='nation_idx', value_name='nation_name').ORDER_BY(region_name, nation_idx)", + "TPCH", + lambda: pd.DataFrame( + { + "region_name": ["AFRICA"] * 5 + + ["AMERICA"] * 5 + + ["ASIA"] * 5 + + ["EUROPE"] * 5 + + ["MIDDLE EAST"] * 5, + "nation_idx": list(range(5)) * 5, + "nation_name": [ + "ALGERIA", + "ETHIOPIA", + "KENYA", + "MOROCCO", + "MOZAMBIQUE", + "ARGENTINA", + "BRAZIL", + "CANADA", + "PERU", + "UNITED STATES", + "CHINA", + "INDIA", + "INDONESIA", + "JAPAN", + "VIETNAM", + "FRANCE", + "GERMANY", + "ROMANIA", + "RUSSIA", + "UNITED KINGDOM", + "EGYPT", + "IRAN", + "IRAQ", + "JORDAN", + "SAUDI ARABIA", + ], + } + ), + "explode_01", + order_sensitive=True, + skipped_dialects={"ANSI", "SQLITE"}, + ), + id="explode_01", + ), pytest.param( PyDoughPandasTest( simple_range_1, diff --git a/tests/testing_utilities.py b/tests/testing_utilities.py index 43207290b..636c593a2 100644 --- a/tests/testing_utilities.py +++ b/tests/testing_utilities.py @@ -1285,6 +1285,12 @@ class PyDoughPandasTest: If True, does not run the test as part of SQL testing. """ + skipped_dialects: set[str] | None = None + """ + If provided, contains the names of all dialects to skip when running the + test in SQL or E2E mode. + """ + fix_output_dialect: str = "sqlite" """ Dialect name to update output @@ -1384,6 +1390,14 @@ def run_sql_test( if self.skip_sql: pytest.skip(f"Skipping SQL text test for {self.test_name}") + if ( + self.skipped_dialects is not None + and database.dialect.name in self.skipped_dialects + ): + pytest.skip( + f"Skipping SQL text test for {self.test_name} on {database.dialect.name} dialect" + ) + # Obtain the graph and the unqualified node graph: GraphMetadata = fetcher(self.graph_name) @@ -1467,6 +1481,14 @@ def run_e2e_test( `table_name_prefix`: Prefix to prepend to table names in to_table calls. Used for Snowflake cross-database writes (e.g., "E2E_TESTS_DB.PUBLIC."). """ + if ( + self.skipped_dialects is not None + and database.dialect.name in self.skipped_dialects + ): + pytest.skip( + f"Skipping E2E test for {self.test_name} on {database.dialect.name} dialect" + ) + # Obtain the graph and the unqualified node graph: GraphMetadata = fetcher(self.graph_name) From a9a5c6c02a7ec0886b4d1ab47a66f29e07655e52 Mon Sep 17 00:00:00 2001 From: knassre-bodo Date: Fri, 15 May 2026 08:38:50 -0700 Subject: [PATCH 02/44] Adding LISTOF operator WIP --- pydough/database_connectors/database_connector.py | 4 ++++ pydough/errors/pydough_error_builder.py | 15 +++++++++++++++ pydough/pydough_operators/__init__.py | 2 ++ .../expression_operators/__init__.py | 2 ++ .../registered_expression_operators.py | 4 ++++ .../transform_bindings/base_transform_bindings.py | 13 +++++++++++++ .../mysql_transform_bindings.py | 5 +++++ .../oracle_transform_bindings.py | 5 +++++ .../postgres_transform_bindings.py | 5 +++++ .../transform_bindings/sf_transform_bindings.py | 5 +++++ tests/test_pipeline_tpch_custom.py | 4 ++-- tests/testing_utilities.py | 5 +++++ 12 files changed, 67 insertions(+), 2 deletions(-) diff --git a/pydough/database_connectors/database_connector.py b/pydough/database_connectors/database_connector.py index e49ca784f..2b33af3bf 100644 --- a/pydough/database_connectors/database_connector.py +++ b/pydough/database_connectors/database_connector.py @@ -65,6 +65,10 @@ def execute_query_df(self, sql: str) -> pd.DataFrame: _ = cast(SnowflakeCursor, self.cursor).fetch_pandas_all # At run-time check and run the fetch. if hasattr(self.cursor, "fetch_pandas_all"): + breakpoint() + # TODO: If type_code for self.cursor.description is 5/9/10 for + # the current column, use json.loads to parse it into the + # appropriate Python type. return self.cursor.fetch_pandas_all() else: # Assume sqlite3 diff --git a/pydough/errors/pydough_error_builder.py b/pydough/errors/pydough_error_builder.py index 0da739bb5..0733c45da 100644 --- a/pydough/errors/pydough_error_builder.py +++ b/pydough/errors/pydough_error_builder.py @@ -255,6 +255,21 @@ def sql_call_conversion_error( f"Failed to convert expression {call.to_string(True)} to SQL: {error}" ) + def sql_call_dialect_unsupported( + self, operator: "PyDoughOperator", dialect: str + ) -> PyDoughException: + """ + Creates an exception for when a SQL dialect does not allow converting + a certain feature to SQL. + + Args: + `operator`: The function operator that is not supported. + `dialect`: The SQL dialect in which the feature is not supported. + """ + return PyDoughSQLException( + f"Cannot convert function {operator} to SQL using dialect {dialect}" + ) + def undefined_function_call( self, node: "UnqualifiedNode", *args, **kwargs ) -> PyDoughException: diff --git a/pydough/pydough_operators/__init__.py b/pydough/pydough_operators/__init__.py index fe6a71a50..5f88494b2 100644 --- a/pydough/pydough_operators/__init__.py +++ b/pydough/pydough_operators/__init__.py @@ -51,6 +51,7 @@ "LEQ", "LET", "LIKE", + "LISTOF", "LOWER", "LPAD", "MAX", @@ -156,6 +157,7 @@ LEQ, LET, LIKE, + LISTOF, LOWER, LPAD, MAX, diff --git a/pydough/pydough_operators/expression_operators/__init__.py b/pydough/pydough_operators/expression_operators/__init__.py index 78f0a60c1..926bbdc47 100644 --- a/pydough/pydough_operators/expression_operators/__init__.py +++ b/pydough/pydough_operators/expression_operators/__init__.py @@ -48,6 +48,7 @@ "LEQ", "LET", "LIKE", + "LISTOF", "LOWER", "LPAD", "MAX", @@ -149,6 +150,7 @@ LEQ, LET, LIKE, + LISTOF, LOWER, LPAD, MAX, diff --git a/pydough/pydough_operators/expression_operators/registered_expression_operators.py b/pydough/pydough_operators/expression_operators/registered_expression_operators.py index b03cf5e4f..fefee60ac 100644 --- a/pydough/pydough_operators/expression_operators/registered_expression_operators.py +++ b/pydough/pydough_operators/expression_operators/registered_expression_operators.py @@ -42,6 +42,7 @@ "LEQ", "LET", "LIKE", + "LISTOF", "LOWER", "LPAD", "MAX", @@ -167,6 +168,9 @@ QUANTILE = ExpressionFunctionOperator( "QUANTILE", True, RequireNumArgs(2), ConstantType(NumericType()) ) +LISTOF = ExpressionFunctionOperator( + "LISTOF", True, RequireNumArgs(1), SelectArgumentType(0) +) POWER = ExpressionFunctionOperator( "POWER", False, RequireNumArgs(2), ConstantType(NumericType()) ) diff --git a/pydough/sqlglot/transform_bindings/base_transform_bindings.py b/pydough/sqlglot/transform_bindings/base_transform_bindings.py index 740102eaa..cbad9370a 100644 --- a/pydough/sqlglot/transform_bindings/base_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/base_transform_bindings.py @@ -262,6 +262,8 @@ def convert_call_to_sqlglot( return sqlglot_expressions.Count( this=sqlglot_expressions.Distinct(expressions=[args[0]]) ) + case pydop.LISTOF: + return self.convert_listof(args, types) case pydop.STARTSWITH: return self.convert_startswith(args, types) case pydop.ENDSWITH: @@ -378,6 +380,17 @@ def convert_sum( """ return sqlglot_expressions.Sum.from_arg_list(args) + def convert_listof( + self, args: SQLGlotExpression, types: list[PyDoughType] + ) -> SQLGlotExpression: + """ + Converts a LISTOF function call to its SQLGlot equivalent. Some dialects + do not support this functionality. + """ + raise self._visitor._session.error_builder.sql_call_dialect_unsupported( + pydop.LISTOF, self._visitor._session.database.dialect.name + ) + def convert_find( self, args: list[SQLGlotExpression], diff --git a/pydough/sqlglot/transform_bindings/mysql_transform_bindings.py b/pydough/sqlglot/transform_bindings/mysql_transform_bindings.py index 6e2f77277..c07625163 100644 --- a/pydough/sqlglot/transform_bindings/mysql_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/mysql_transform_bindings.py @@ -88,6 +88,11 @@ def convert_call_to_sqlglot( return super().convert_call_to_sqlglot(operator, args, types) + def convert_listof( + self, args: SQLGlotExpression, types: list[PyDoughType] + ) -> SQLGlotExpression: + return sqlglot_expressions.Anonymous(this="JSON_ARRAYAGG", expressions=args) + def convert_slice( self, args: list[SQLGlotExpression], types: list[PyDoughType] ) -> SQLGlotExpression: diff --git a/pydough/sqlglot/transform_bindings/oracle_transform_bindings.py b/pydough/sqlglot/transform_bindings/oracle_transform_bindings.py index 54880ceab..87ec8fd02 100644 --- a/pydough/sqlglot/transform_bindings/oracle_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/oracle_transform_bindings.py @@ -97,6 +97,11 @@ def convert_call_to_sqlglot( return super().convert_call_to_sqlglot(operator, args, types) + def convert_listof( + self, args: SQLGlotExpression, types: list[PyDoughType] + ) -> SQLGlotExpression: + return sqlglot_expressions.Anonymous(this="COLLECT", expressions=args) + def convert_default_to( self, args: list[SQLGlotExpression], types: list[PyDoughType] ) -> SQLGlotExpression: diff --git a/pydough/sqlglot/transform_bindings/postgres_transform_bindings.py b/pydough/sqlglot/transform_bindings/postgres_transform_bindings.py index 10091450e..6f1937252 100644 --- a/pydough/sqlglot/transform_bindings/postgres_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/postgres_transform_bindings.py @@ -66,6 +66,11 @@ def convert_call_to_sqlglot( return super().convert_call_to_sqlglot(operator, args, types) + def convert_listof( + self, args: SQLGlotExpression, types: list[PyDoughType] + ) -> SQLGlotExpression: + return sqlglot_expressions.Anonymous(this="ARRAY_AGG", expressions=args) + def convert_sum( self, arg: list[SQLGlotExpression], types: list[PyDoughType] ) -> SQLGlotExpression: diff --git a/pydough/sqlglot/transform_bindings/sf_transform_bindings.py b/pydough/sqlglot/transform_bindings/sf_transform_bindings.py index 20e944caf..6307db125 100644 --- a/pydough/sqlglot/transform_bindings/sf_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/sf_transform_bindings.py @@ -85,6 +85,11 @@ def convert_sum( # For other types, use SUM directly return sqlglot_expressions.Sum(this=arg[0]) + def convert_listof( + self, args: SQLGlotExpression, types: list[PyDoughType] + ) -> SQLGlotExpression: + return sqlglot_expressions.ArrayAgg(this=args[0]) + def convert_integer( self, args: list[SQLGlotExpression], types: list[PyDoughType] ) -> SQLGlotExpression: diff --git a/tests/test_pipeline_tpch_custom.py b/tests/test_pipeline_tpch_custom.py index b05632bc3..b65c30329 100644 --- a/tests/test_pipeline_tpch_custom.py +++ b/tests/test_pipeline_tpch_custom.py @@ -4073,7 +4073,7 @@ ), pytest.param( PyDoughPandasTest( - "result = regions.CALCULATE(region_name=name, nation_names=ARRAY_COLLECT(nations.name)).ORDER_BY(region_name)", + "result = regions.CALCULATE(region_name=name, nation_names=LISTOF(nations.name)).ORDER_BY(region_name)", "TPCH", lambda: pd.DataFrame( { @@ -4107,7 +4107,7 @@ ), pytest.param( PyDoughPandasTest( - "array_data = regions.CALCULATE(nation_names=ARRAY_COLLECT(nations.name))\n" + "array_data = regions.CALCULATE(nation_names=LISTOF(nations.name))\n" "result = array_data.EXPLODE(nation_names, index_name='nation_idx', value_name='nation_name').ORDER_BY(region_name, nation_idx)", "TPCH", lambda: pd.DataFrame( diff --git a/tests/testing_utilities.py b/tests/testing_utilities.py index 636c593a2..178cfa7e6 100644 --- a/tests/testing_utilities.py +++ b/tests/testing_utilities.py @@ -1557,6 +1557,11 @@ def run_e2e_test( result[col_name], refsol[col_name] ) + print() + print(result.to_string()) + print(refsol.to_string()) + breakpoint() + # Perform the comparison between the result and the reference solution pd.testing.assert_frame_equal( result, refsol, check_dtype=(not coerce_types), check_exact=False, atol=1e-8 From c7ec035c336a4fb555ae2628c1e41275980521b5 Mon Sep 17 00:00:00 2001 From: knassre-bodo Date: Fri, 22 May 2026 11:06:08 -0700 Subject: [PATCH 03/44] Fixed snowflake array data returning via json.loads --- pydough/database_connectors/database_connector.py | 14 +++++++++----- tests/test_pipeline_tpch_custom.py | 2 +- tests/testing_utilities.py | 5 ----- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/pydough/database_connectors/database_connector.py b/pydough/database_connectors/database_connector.py index 2b33af3bf..447c2dc69 100644 --- a/pydough/database_connectors/database_connector.py +++ b/pydough/database_connectors/database_connector.py @@ -11,6 +11,7 @@ "DatabaseDialect", ] +import json from dataclasses import dataclass from enum import Enum from typing import TYPE_CHECKING, Union, cast @@ -65,11 +66,14 @@ def execute_query_df(self, sql: str) -> pd.DataFrame: _ = cast(SnowflakeCursor, self.cursor).fetch_pandas_all # At run-time check and run the fetch. if hasattr(self.cursor, "fetch_pandas_all"): - breakpoint() - # TODO: If type_code for self.cursor.description is 5/9/10 for - # the current column, use json.loads to parse it into the - # appropriate Python type. - return self.cursor.fetch_pandas_all() + pd_table: pd.DataFrame = self.cursor.fetch_pandas_all() + # For each column returned with types 5/9/10 (Snowflake's type + # codes for array/object/variant), parse the JSON string into + # the appropriate Python type using json.loads. + for idx, dtype in enumerate(self.cursor.description): + if dtype[1] in (5, 9, 10): + pd_table.iloc[:, idx] = pd_table.iloc[:, idx].apply(json.loads) + return pd_table else: # Assume sqlite3 column_names: list[str] = [ diff --git a/tests/test_pipeline_tpch_custom.py b/tests/test_pipeline_tpch_custom.py index b65c30329..2e78b7aa1 100644 --- a/tests/test_pipeline_tpch_custom.py +++ b/tests/test_pipeline_tpch_custom.py @@ -4087,7 +4087,7 @@ "nation_names": [ ["ALGERIA", "ETHIOPIA", "KENYA", "MOROCCO", "MOZAMBIQUE"], ["ARGENTINA", "BRAZIL", "CANADA", "PERU", "UNITED STATES"], - ["CHINA", "INDIA", "INDONESIA", "JAPAN", "VIETNAM"], + ["INDIA", "INDONESIA", "JAPAN", "CHINA", "VIETNAM"], [ "FRANCE", "GERMANY", diff --git a/tests/testing_utilities.py b/tests/testing_utilities.py index 178cfa7e6..636c593a2 100644 --- a/tests/testing_utilities.py +++ b/tests/testing_utilities.py @@ -1557,11 +1557,6 @@ def run_e2e_test( result[col_name], refsol[col_name] ) - print() - print(result.to_string()) - print(refsol.to_string()) - breakpoint() - # Perform the comparison between the result and the reference solution pd.testing.assert_frame_equal( result, refsol, check_dtype=(not coerce_types), check_exact=False, atol=1e-8 From baf47cb18e2fe6a7564dea98c55c929cc5f10900 Mon Sep 17 00:00:00 2001 From: knassre-bodo Date: Wed, 27 May 2026 10:04:55 -0700 Subject: [PATCH 04/44] WIP --- tests/test_pipeline_tpch_custom.py | 2 +- tests/test_plan_refsols/array_data_01.txt | 5 +++++ tests/test_sql_refsols/array_data_01_mysql.sql | 10 ++++++++++ tests/test_sql_refsols/array_data_01_oracle.sql | 10 ++++++++++ .../test_sql_refsols/array_data_01_postgres.sql | 16 ++++++++++++++++ .../test_sql_refsols/array_data_01_snowflake.sql | 16 ++++++++++++++++ 6 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 tests/test_plan_refsols/array_data_01.txt create mode 100644 tests/test_sql_refsols/array_data_01_mysql.sql create mode 100644 tests/test_sql_refsols/array_data_01_oracle.sql create mode 100644 tests/test_sql_refsols/array_data_01_postgres.sql create mode 100644 tests/test_sql_refsols/array_data_01_snowflake.sql diff --git a/tests/test_pipeline_tpch_custom.py b/tests/test_pipeline_tpch_custom.py index 2e78b7aa1..16aa148bf 100644 --- a/tests/test_pipeline_tpch_custom.py +++ b/tests/test_pipeline_tpch_custom.py @@ -4129,10 +4129,10 @@ "CANADA", "PERU", "UNITED STATES", - "CHINA", "INDIA", "INDONESIA", "JAPAN", + "CHINA", "VIETNAM", "FRANCE", "GERMANY", diff --git a/tests/test_plan_refsols/array_data_01.txt b/tests/test_plan_refsols/array_data_01.txt new file mode 100644 index 000000000..c1b9e719d --- /dev/null +++ b/tests/test_plan_refsols/array_data_01.txt @@ -0,0 +1,5 @@ +ROOT(columns=[('region_name', r_name), ('nation_names', listof_n_name)], orderings=[(r_name):asc_first]) + JOIN(condition=t0.r_regionkey == t1.n_regionkey, type=INNER, cardinality=SINGULAR_ACCESS, reverse_cardinality=SINGULAR_ACCESS, columns={'listof_n_name': t1.listof_n_name, 'r_name': t0.r_name}) + SCAN(table=tpch.REGION, columns={'r_name': r_name, 'r_regionkey': r_regionkey}) + AGGREGATE(keys={'n_regionkey': n_regionkey}, aggregations={'listof_n_name': LISTOF(n_name)}) + SCAN(table=tpch.NATION, columns={'n_name': n_name, 'n_regionkey': n_regionkey}) diff --git a/tests/test_sql_refsols/array_data_01_mysql.sql b/tests/test_sql_refsols/array_data_01_mysql.sql new file mode 100644 index 000000000..513d637da --- /dev/null +++ b/tests/test_sql_refsols/array_data_01_mysql.sql @@ -0,0 +1,10 @@ +SELECT + REGION.r_name COLLATE utf8mb4_bin AS region_name, + JSON_ARRAYAGG(NATION.n_name) AS nation_names +FROM tpch.REGION AS REGION +JOIN tpch.NATION AS NATION + ON NATION.n_regionkey = REGION.r_regionkey +GROUP BY + NATION.n_regionkey +ORDER BY + 1 diff --git a/tests/test_sql_refsols/array_data_01_oracle.sql b/tests/test_sql_refsols/array_data_01_oracle.sql new file mode 100644 index 000000000..877026d89 --- /dev/null +++ b/tests/test_sql_refsols/array_data_01_oracle.sql @@ -0,0 +1,10 @@ +SELECT + REGION.r_name AS region_name, + COLLECT(NATION.n_name) AS nation_names +FROM TPCH.REGION REGION +JOIN TPCH.NATION NATION + ON NATION.n_regionkey = REGION.r_regionkey +GROUP BY + NATION.n_regionkey +ORDER BY + 1 NULLS FIRST diff --git a/tests/test_sql_refsols/array_data_01_postgres.sql b/tests/test_sql_refsols/array_data_01_postgres.sql new file mode 100644 index 000000000..c1532d222 --- /dev/null +++ b/tests/test_sql_refsols/array_data_01_postgres.sql @@ -0,0 +1,16 @@ +WITH _s1 AS ( + SELECT + ARRAY_AGG(n_name) AS listof_n_name, + n_regionkey + FROM tpch.nation + GROUP BY + 2 +) +SELECT + region.r_name AS region_name, + _s1.listof_n_name AS nation_names +FROM tpch.region AS region +JOIN _s1 AS _s1 + ON _s1.n_regionkey = region.r_regionkey +ORDER BY + 1 NULLS FIRST diff --git a/tests/test_sql_refsols/array_data_01_snowflake.sql b/tests/test_sql_refsols/array_data_01_snowflake.sql new file mode 100644 index 000000000..380a2ce27 --- /dev/null +++ b/tests/test_sql_refsols/array_data_01_snowflake.sql @@ -0,0 +1,16 @@ +WITH _s1 AS ( + SELECT + n_regionkey, + ARRAY_AGG(n_name) AS listof_n_name + FROM tpch.nation + GROUP BY + 1 +) +SELECT + region.r_name AS region_name, + _s1.listof_n_name AS nation_names +FROM tpch.region AS region +JOIN _s1 AS _s1 + ON _s1.n_regionkey = region.r_regionkey +ORDER BY + 1 NULLS FIRST From 91a18cc0c8b994fbc676aea72ac97dc89c569077 Mon Sep 17 00:00:00 2001 From: knassre-bodo Date: Fri, 29 May 2026 10:02:10 -0700 Subject: [PATCH 05/44] Experimenting with ordering --- .../database_connectors/builtin_databases.py | 18 ++++---- .../database_connectors/database_connector.py | 45 +++++++++++++++---- .../database_connectors/empty_connection.py | 6 ++- .../mysql_transform_bindings.py | 8 +++- tests/conftest.py | 36 +++++++++++---- .../test_sql_refsols/array_data_01_mysql.sql | 2 +- 6 files changed, 85 insertions(+), 30 deletions(-) diff --git a/pydough/database_connectors/builtin_databases.py b/pydough/database_connectors/builtin_databases.py index 5bac56d03..bea129003 100644 --- a/pydough/database_connectors/builtin_databases.py +++ b/pydough/database_connectors/builtin_databases.py @@ -86,7 +86,7 @@ def load_sqlite_connection(**kwargs) -> DatabaseConnection: if "database" not in kwargs: raise PyDoughSessionException("SQLite connection requires a database path.") connection: sqlite3.Connection = sqlite3.connect(**kwargs) - return DatabaseConnection(connection) + return DatabaseConnection(connection, DatabaseDialect.SQLITE) def load_snowflake_connection(**kwargs) -> DatabaseConnection: @@ -118,7 +118,7 @@ def load_snowflake_connection(**kwargs) -> DatabaseConnection: connection: snowflake.connector.connection.SnowflakeConnection if connection := kwargs.pop("connection", None): # If a connection object is provided, return it wrapped in DatabaseConnection - return DatabaseConnection(connection) + return DatabaseConnection(connection, DatabaseDialect.SNOWFLAKE) # Snowflake connection requires specific parameters: # user, password, account. # Raise an error if any of these are missing. @@ -133,7 +133,7 @@ def load_snowflake_connection(**kwargs) -> DatabaseConnection: ) # Create a Snowflake connection using the provided keyword arguments connection = snowflake.connector.connect(**kwargs) - return DatabaseConnection(connection) + return DatabaseConnection(connection, DatabaseDialect.SNOWFLAKE) def load_mysql_connection(**kwargs) -> DatabaseConnection: @@ -178,7 +178,7 @@ def load_mysql_connection(**kwargs) -> DatabaseConnection: if connection := kwargs.pop("connection", None): # If a connection object is provided, return it wrapped in # DatabaseConnection - return DatabaseConnection(connection) + return DatabaseConnection(connection, DatabaseDialect.MYSQL) # MySQL connection requires specific parameters: # user, password, database. @@ -213,7 +213,7 @@ def load_mysql_connection(**kwargs) -> DatabaseConnection: while attempt <= attempts: try: connection = mysql.connector.connect(**kwargs) - return DatabaseConnection(connection) + return DatabaseConnection(connection, DatabaseDialect.MYSQL) except (OSError, mysql.connector.Error) as err: if attempt >= attempts: @@ -269,7 +269,7 @@ def load_postgres_connection(**kwargs) -> DatabaseConnection: if connection := kwargs.pop("connection", None): # If a connection object is provided, return it wrapped in # DatabaseConnection - return DatabaseConnection(connection) + return DatabaseConnection(connection, DatabaseDialect.POSTGRES) # Postgres connection requires specific parameters: # user, password, dbname. @@ -303,7 +303,7 @@ def load_postgres_connection(**kwargs) -> DatabaseConnection: while attempt <= attempts: try: connection = psycopg2.connect(**kwargs) - return DatabaseConnection(connection) + return DatabaseConnection(connection, DatabaseDialect.POSTGRES) except (OSError, psycopg2.Error) as err: if attempt >= attempts: @@ -357,7 +357,7 @@ def load_oracle_connection(**kwargs) -> DatabaseConnection: # If a connection object is provided, return it wrapped in # DatabaseConnection assert isinstance(connection, oracledb.Connection) - return DatabaseConnection(connection) + return DatabaseConnection(connection, DatabaseDialect.ORACLE) # Oracle connection requires specific parameters: # user, password, host and service_name. @@ -395,7 +395,7 @@ def load_oracle_connection(**kwargs) -> DatabaseConnection: while attempt <= attempts: try: connection = oracledb.connect(**kwargs) - return DatabaseConnection(connection) + return DatabaseConnection(connection, DatabaseDialect.ORACLE) except (OSError, oracledb.Error) as err: if attempt >= attempts: diff --git a/pydough/database_connectors/database_connector.py b/pydough/database_connectors/database_connector.py index 447c2dc69..4eead1d35 100644 --- a/pydough/database_connectors/database_connector.py +++ b/pydough/database_connectors/database_connector.py @@ -37,10 +37,12 @@ class DatabaseConnection: # with Python. _connection: DBConnection _cursor: DBCursor | None + _dialect: "DatabaseDialect" - def __init__(self, connection: DBConnection) -> None: + def __init__(self, connection: DBConnection, dialect: "DatabaseDialect") -> None: self._connection = connection self._cursor = None + self._dialect = dialect def execute_query_df(self, sql: str) -> pd.DataFrame: """Create a cursor object using the connection and execute the query, @@ -64,15 +66,39 @@ def execute_query_df(self, sql: str) -> pd.DataFrame: # check at run-time if the cursor has the method. if TYPE_CHECKING: _ = cast(SnowflakeCursor, self.cursor).fetch_pandas_all + + # Identify which columns must be coerced from strings into native + # types using json.loads based on the cursor description. + semi_structured_cols: set[int] = set() + match self._dialect: + case DatabaseDialect.SNOWFLAKE: + # Snowflake returns array/object/variant types as JSON + # strings (types 5/9/10), so we need to parse those back + # into Python types. + semi_structured_cols.update( + { + idx + for idx, desc in enumerate(self.cursor.description) + if desc[1] in (5, 9, 10) + } + ) + case DatabaseDialect.MYSQL: + # Snowflake returns JSON data (type 245) as strings, so we + # need to parse those back into Python types. + semi_structured_cols.update( + { + idx + for idx, desc in enumerate(self.cursor.description) + if desc[1] == 245 + } + ) + case _: + pass # At run-time check and run the fetch. if hasattr(self.cursor, "fetch_pandas_all"): pd_table: pd.DataFrame = self.cursor.fetch_pandas_all() - # For each column returned with types 5/9/10 (Snowflake's type - # codes for array/object/variant), parse the JSON string into - # the appropriate Python type using json.loads. - for idx, dtype in enumerate(self.cursor.description): - if dtype[1] in (5, 9, 10): - pd_table.iloc[:, idx] = pd_table.iloc[:, idx].apply(json.loads) + for idx in semi_structured_cols: + pd_table.iloc[:, idx] = pd_table.iloc[:, idx].apply(json.loads) return pd_table else: # Assume sqlite3 @@ -82,7 +108,10 @@ def execute_query_df(self, sql: str) -> pd.DataFrame: # TODO: (gh #174) Cache the cursor? # TODO: (gh #175) enable typed DataFrames. data = self.cursor.fetchall() - return pd.DataFrame(data, columns=column_names) + pd_table = pd.DataFrame(data, columns=column_names) + for idx in semi_structured_cols: + pd_table.iloc[:, idx] = pd_table.iloc[:, idx].apply(json.loads) + return pd_table except Exception as e: print(f"ERROR WHILE EXECUTING QUERY:\n{sql}") raise pydough.active_session.error_builder.sql_runtime_failure( diff --git a/pydough/database_connectors/empty_connection.py b/pydough/database_connectors/empty_connection.py index 384c94d65..c29b9ffa8 100644 --- a/pydough/database_connectors/empty_connection.py +++ b/pydough/database_connectors/empty_connection.py @@ -11,7 +11,7 @@ from pydough.errors import PyDoughSessionException -from .database_connector import DatabaseConnection +from .database_connector import DatabaseConnection, DatabaseDialect class EmptyConnection(Connection): @@ -36,4 +36,6 @@ def cursor(self, *args, **kwargs): raise PyDoughSessionException("No SQL Database is specified.") -empty_connection: DatabaseConnection = DatabaseConnection(EmptyConnection()) +empty_connection: DatabaseConnection = DatabaseConnection( + EmptyConnection(), DatabaseDialect.ANSI +) diff --git a/pydough/sqlglot/transform_bindings/mysql_transform_bindings.py b/pydough/sqlglot/transform_bindings/mysql_transform_bindings.py index c07625163..706e7e394 100644 --- a/pydough/sqlglot/transform_bindings/mysql_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/mysql_transform_bindings.py @@ -91,7 +91,13 @@ def convert_call_to_sqlglot( def convert_listof( self, args: SQLGlotExpression, types: list[PyDoughType] ) -> SQLGlotExpression: - return sqlglot_expressions.Anonymous(this="JSON_ARRAYAGG", expressions=args) + order_exp = sqlglot_expressions.Order( + this=args[0], + expressions=[sqlglot_expressions.Ordered(this=args[0], nulls_first=True)], + ) + return sqlglot_expressions.Anonymous( + this="JSON_ARRAYAGG", expressions=[order_exp] + ) def convert_slice( self, args: list[SQLGlotExpression], types: list[PyDoughType] diff --git a/tests/conftest.py b/tests/conftest.py index a84f7dbbf..d674f28a8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -544,7 +544,7 @@ def sqlite_people_jobs() -> DatabaseConnection: ) """ sqlite3_empty_connection: DatabaseConnection = DatabaseConnection( - sqlite3.connect(":memory:") + sqlite3.connect(":memory:"), DatabaseDialect.SQLITE ) cursor: sqlite3.Cursor = sqlite3_empty_connection.connection.cursor() cursor.execute(create_table_1) @@ -605,7 +605,10 @@ def sqlite_tpch_db_context(sqlite_tpch_db) -> DatabaseContext: """ Return a DatabaseContext for the SQLite TPCH database. """ - return DatabaseContext(DatabaseConnection(sqlite_tpch_db), DatabaseDialect.SQLITE) + return DatabaseContext( + DatabaseConnection(sqlite_tpch_db, DatabaseDialect.SQLITE), + DatabaseDialect.SQLITE, + ) @pytest.fixture @@ -752,7 +755,9 @@ def sqlite_defog_connection() -> DatabaseContext: subprocess.run("cd tests/gen_data; bash setup_defog.sh", shell=True) path: str = os.path.join(base_dir, "tests/gen_data/defog.db") connection: sqlite3.Connection = sqlite3.connect(path) - return DatabaseContext(DatabaseConnection(connection), DatabaseDialect.SQLITE) + return DatabaseContext( + DatabaseConnection(connection, DatabaseDialect.SQLITE), DatabaseDialect.SQLITE + ) @pytest.fixture(scope="session") @@ -769,7 +774,9 @@ def sqlite_epoch_connection() -> DatabaseContext: ) path: str = os.path.join(base_dir, "tests/gen_data/epoch.db") connection: sqlite3.Connection = sqlite3.connect(path) - return DatabaseContext(DatabaseConnection(connection), DatabaseDialect.SQLITE) + return DatabaseContext( + DatabaseConnection(connection, DatabaseDialect.SQLITE), DatabaseDialect.SQLITE + ) @pytest.fixture(scope="session") @@ -793,7 +800,9 @@ def sqlite_technograph_connection() -> DatabaseContext: gen_technograph_records(cursor) # Return the database context. - return DatabaseContext(DatabaseConnection(connection), DatabaseDialect.SQLITE) + return DatabaseContext( + DatabaseConnection(connection, DatabaseDialect.SQLITE), DatabaseDialect.SQLITE + ) @pytest.fixture( @@ -829,7 +838,9 @@ def sqlite_cryptbank_connection() -> DatabaseContext: path: str = os.path.join(base_dir, "tests/gen_data/cryptbank.db") connection: sqlite3.Connection = sqlite3.connect(":memory:") connection.execute(f"attach database '{path}' as CRBNK") - return DatabaseContext(DatabaseConnection(connection), DatabaseDialect.SQLITE) + return DatabaseContext( + DatabaseConnection(connection, DatabaseDialect.SQLITE), DatabaseDialect.SQLITE + ) @pytest.fixture(scope="session") @@ -865,7 +876,10 @@ def _impl(database_name: str) -> DatabaseContext: connection = sqlite3.connect(":memory:") connection.execute(f"ATTACH DATABASE '{file_path}' AS {database_name}") - return DatabaseContext(DatabaseConnection(connection), DatabaseDialect.SQLITE) + return DatabaseContext( + DatabaseConnection(connection, DatabaseDialect.SQLITE), + DatabaseDialect.SQLITE, + ) return _impl @@ -1047,7 +1061,10 @@ def _impl(database_name: str) -> DatabaseContext: file_path: str = os.path.join(full_dir_path, f"{database_name}.db") connection = sqlite3.connect(file_path) - return DatabaseContext(DatabaseConnection(connection), DatabaseDialect.SQLITE) + return DatabaseContext( + DatabaseConnection(connection, DatabaseDialect.SQLITE), + DatabaseDialect.SQLITE, + ) return _impl @@ -1684,7 +1701,8 @@ def sqlite_pagerank_db_contexts() -> dict[str, DatabaseContext]: # database context in the result. gen_pagerank_records(connection, nodes, edges) result[name] = DatabaseContext( - DatabaseConnection(connection), DatabaseDialect.SQLITE + DatabaseConnection(connection, DatabaseDialect.SQLITE), + DatabaseDialect.SQLITE, ) return result diff --git a/tests/test_sql_refsols/array_data_01_mysql.sql b/tests/test_sql_refsols/array_data_01_mysql.sql index 513d637da..8e0ede9ff 100644 --- a/tests/test_sql_refsols/array_data_01_mysql.sql +++ b/tests/test_sql_refsols/array_data_01_mysql.sql @@ -1,6 +1,6 @@ SELECT REGION.r_name COLLATE utf8mb4_bin AS region_name, - JSON_ARRAYAGG(NATION.n_name) AS nation_names + JSON_ARRAYAGG(NATION.n_name ORDER BY NATION.n_name) AS nation_names FROM tpch.REGION AS REGION JOIN tpch.NATION AS NATION ON NATION.n_regionkey = REGION.r_regionkey From 6a1d179a37bef2469f01631aaf1f3f4f5c160b61 Mon Sep 17 00:00:00 2001 From: knassre-bodo Date: Tue, 2 Jun 2026 12:38:43 -0700 Subject: [PATCH 06/44] Fixing array_agg testing for dialects --- .../database_connectors/builtin_databases.py | 4 ++-- .../mysql_transform_bindings.py | 8 +------- .../oracle_transform_bindings.py | 2 +- .../trino_transform_bindings.py | 5 +++++ tests/test_pipeline_tpch_custom.py | 1 + tests/test_sql_refsols/array_data_01_mysql.sql | 2 +- tests/test_sql_refsols/array_data_01_oracle.sql | 2 +- tests/test_sql_refsols/array_data_01_trino.sql | 16 ++++++++++++++++ tests/testing_utilities.py | 17 +++++++++++++++++ 9 files changed, 45 insertions(+), 12 deletions(-) create mode 100644 tests/test_sql_refsols/array_data_01_trino.sql diff --git a/pydough/database_connectors/builtin_databases.py b/pydough/database_connectors/builtin_databases.py index 75c1a3937..7f7d9e0df 100644 --- a/pydough/database_connectors/builtin_databases.py +++ b/pydough/database_connectors/builtin_databases.py @@ -168,7 +168,7 @@ def load_trino_connection(**kwargs) -> DatabaseConnection: connection = kwargs.pop("connection", None) if connection: - return DatabaseConnection(connection) + return DatabaseConnection(connection, dialect=DatabaseDialect.TRINO) required_keys = ["user", "host", "port"] if not all(key in kwargs for key in required_keys): raise ValueError( @@ -177,7 +177,7 @@ def load_trino_connection(**kwargs) -> DatabaseConnection: ) # Create a Trino connection using the provided keyword arguments connection = trino.dbapi.connect(**kwargs) - return DatabaseConnection(connection) + return DatabaseConnection(connection, dialect=DatabaseDialect.TRINO) def load_mysql_connection(**kwargs) -> DatabaseConnection: diff --git a/pydough/sqlglot/transform_bindings/mysql_transform_bindings.py b/pydough/sqlglot/transform_bindings/mysql_transform_bindings.py index 706e7e394..c07625163 100644 --- a/pydough/sqlglot/transform_bindings/mysql_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/mysql_transform_bindings.py @@ -91,13 +91,7 @@ def convert_call_to_sqlglot( def convert_listof( self, args: SQLGlotExpression, types: list[PyDoughType] ) -> SQLGlotExpression: - order_exp = sqlglot_expressions.Order( - this=args[0], - expressions=[sqlglot_expressions.Ordered(this=args[0], nulls_first=True)], - ) - return sqlglot_expressions.Anonymous( - this="JSON_ARRAYAGG", expressions=[order_exp] - ) + return sqlglot_expressions.Anonymous(this="JSON_ARRAYAGG", expressions=args) def convert_slice( self, args: list[SQLGlotExpression], types: list[PyDoughType] diff --git a/pydough/sqlglot/transform_bindings/oracle_transform_bindings.py b/pydough/sqlglot/transform_bindings/oracle_transform_bindings.py index 87ec8fd02..4afeaa1f2 100644 --- a/pydough/sqlglot/transform_bindings/oracle_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/oracle_transform_bindings.py @@ -100,7 +100,7 @@ def convert_call_to_sqlglot( def convert_listof( self, args: SQLGlotExpression, types: list[PyDoughType] ) -> SQLGlotExpression: - return sqlglot_expressions.Anonymous(this="COLLECT", expressions=args) + return sqlglot_expressions.Anonymous(this="JSON_ARRAYAGG", expressions=args) def convert_default_to( self, args: list[SQLGlotExpression], types: list[PyDoughType] diff --git a/pydough/sqlglot/transform_bindings/trino_transform_bindings.py b/pydough/sqlglot/transform_bindings/trino_transform_bindings.py index a227fca91..ef5764fa9 100644 --- a/pydough/sqlglot/transform_bindings/trino_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/trino_transform_bindings.py @@ -126,6 +126,11 @@ def convert_extract_datetime( ) return func_expr + def convert_listof( + self, args: SQLGlotExpression, types: list[PyDoughType] + ) -> SQLGlotExpression: + return sqlglot_expressions.ArrayAgg(this=args[0]) + def convert_datediff( self, args: list[SQLGlotExpression], diff --git a/tests/test_pipeline_tpch_custom.py b/tests/test_pipeline_tpch_custom.py index d1ab9f798..2c99007f2 100644 --- a/tests/test_pipeline_tpch_custom.py +++ b/tests/test_pipeline_tpch_custom.py @@ -4107,6 +4107,7 @@ ), "array_data_01", order_sensitive=True, + ignore_array_order=True, skipped_dialects={"ANSI", "SQLITE"}, ), id="array_data_01", diff --git a/tests/test_sql_refsols/array_data_01_mysql.sql b/tests/test_sql_refsols/array_data_01_mysql.sql index 8e0ede9ff..513d637da 100644 --- a/tests/test_sql_refsols/array_data_01_mysql.sql +++ b/tests/test_sql_refsols/array_data_01_mysql.sql @@ -1,6 +1,6 @@ SELECT REGION.r_name COLLATE utf8mb4_bin AS region_name, - JSON_ARRAYAGG(NATION.n_name ORDER BY NATION.n_name) AS nation_names + JSON_ARRAYAGG(NATION.n_name) AS nation_names FROM tpch.REGION AS REGION JOIN tpch.NATION AS NATION ON NATION.n_regionkey = REGION.r_regionkey diff --git a/tests/test_sql_refsols/array_data_01_oracle.sql b/tests/test_sql_refsols/array_data_01_oracle.sql index 877026d89..fe3351ff9 100644 --- a/tests/test_sql_refsols/array_data_01_oracle.sql +++ b/tests/test_sql_refsols/array_data_01_oracle.sql @@ -1,6 +1,6 @@ SELECT REGION.r_name AS region_name, - COLLECT(NATION.n_name) AS nation_names + JSON_ARRAYAGG(NATION.n_name) AS nation_names FROM TPCH.REGION REGION JOIN TPCH.NATION NATION ON NATION.n_regionkey = REGION.r_regionkey diff --git a/tests/test_sql_refsols/array_data_01_trino.sql b/tests/test_sql_refsols/array_data_01_trino.sql new file mode 100644 index 000000000..380a2ce27 --- /dev/null +++ b/tests/test_sql_refsols/array_data_01_trino.sql @@ -0,0 +1,16 @@ +WITH _s1 AS ( + SELECT + n_regionkey, + ARRAY_AGG(n_name) AS listof_n_name + FROM tpch.nation + GROUP BY + 1 +) +SELECT + region.r_name AS region_name, + _s1.listof_n_name AS nation_names +FROM tpch.region AS region +JOIN _s1 AS _s1 + ON _s1.n_regionkey = region.r_regionkey +ORDER BY + 1 NULLS FIRST diff --git a/tests/testing_utilities.py b/tests/testing_utilities.py index 793142ed0..8c6d71b2a 100644 --- a/tests/testing_utilities.py +++ b/tests/testing_utilities.py @@ -1293,6 +1293,12 @@ class PyDoughPandasTest: test in SQL or E2E mode. """ + ignore_array_order: bool = False + """ + If True, when comparing results, ignores order of elements within array + columns. + """ + fix_output_dialect: str = "sqlite" """ Dialect name to update output @@ -1563,6 +1569,17 @@ def run_e2e_test( result[col_name], refsol[col_name] ) + # Internally sort any array columns so there is no ambiguity in ordering + # within arrays caused by how different dialects group array values. + if self.ignore_array_order and len(result) > 1 and len(refsol) > 1: + for col in result.columns: + result[col] = result[col].apply( + lambda x: sorted(x) if isinstance(x, list) else x + ) + refsol[col] = refsol[col].apply( + lambda x: sorted(x) if isinstance(x, list) else x + ) + # Perform the comparison between the result and the reference solution pd.testing.assert_frame_equal( result, From 6c89f4e79bac50d4b6deced544ad8eb51e3096f5 Mon Sep 17 00:00:00 2001 From: knassre-bodo Date: Tue, 2 Jun 2026 13:43:54 -0700 Subject: [PATCH 07/44] WIP array literal generation on dialects --- .../base_transform_bindings.py | 28 ++++ .../mysql_transform_bindings.py | 5 + .../postgres_transform_bindings.py | 10 ++ .../sf_transform_bindings.py | 5 + .../trino_transform_bindings.py | 5 + .../user_collections/dataframe_collection.py | 15 +- tests/test_pipeline_tpch_custom.py | 158 +++++++++++++++++- tests/test_plan_refsols/array_data_02.txt | 2 + tests/test_plan_refsols/array_data_04.txt | 2 + .../test_sql_refsols/array_data_02_mysql.sql | 9 + .../array_data_02_postgres.sql | 8 + .../array_data_02_snowflake.sql | 8 + .../test_sql_refsols/array_data_02_trino.sql | 11 ++ .../test_sql_refsols/array_data_04_mysql.sql | 25 +++ .../array_data_04_postgres.sql | 21 +++ .../array_data_04_snowflake.sql | 24 +++ .../test_sql_refsols/array_data_04_trino.sql | 21 +++ 17 files changed, 350 insertions(+), 7 deletions(-) create mode 100644 tests/test_plan_refsols/array_data_02.txt create mode 100644 tests/test_plan_refsols/array_data_04.txt create mode 100644 tests/test_sql_refsols/array_data_02_mysql.sql create mode 100644 tests/test_sql_refsols/array_data_02_postgres.sql create mode 100644 tests/test_sql_refsols/array_data_02_snowflake.sql create mode 100644 tests/test_sql_refsols/array_data_02_trino.sql create mode 100644 tests/test_sql_refsols/array_data_04_mysql.sql create mode 100644 tests/test_sql_refsols/array_data_04_postgres.sql create mode 100644 tests/test_sql_refsols/array_data_04_snowflake.sql create mode 100644 tests/test_sql_refsols/array_data_04_trino.sql diff --git a/pydough/sqlglot/transform_bindings/base_transform_bindings.py b/pydough/sqlglot/transform_bindings/base_transform_bindings.py index 174b0518f..005f5f648 100644 --- a/pydough/sqlglot/transform_bindings/base_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/base_transform_bindings.py @@ -23,11 +23,13 @@ LiteralExpression, ) from pydough.types import ( + ArrayType, BooleanType, DatetimeType, NumericType, PyDoughType, StringType, + UnknownType, ) from pydough.user_collections.dataframe_collection import DataframeGeneratedCollection from pydough.user_collections.range_collection import RangeGeneratedCollection @@ -2464,6 +2466,16 @@ def generate_dataframe_item_expression( (None, NaN, BooleanType, StringType) meaning that this representation works through all current dialects. """ + if isinstance(item, list): + inner_type: PyDoughType + if isinstance(item_type, ArrayType): + inner_type = item_type.elem_type + else: + inner_type = UnknownType() + inner_items: list[SQLGlotExpression] = [ + self.generate_dataframe_item_expression(i, inner_type) for i in item + ] + return self.generate_dataframe_array_expression(inner_items, inner_type) if item is None or pd.isna(item): return sqlglot_expressions.Null() @@ -2478,6 +2490,22 @@ def generate_dataframe_item_expression( case _: # Specific dialect expression return self.generate_dataframe_item_dialect_expression(item, item_type) + def generate_dataframe_array_expression( + self, items: list[SQLGlotExpression], inner_type: PyDoughType + ) -> SQLGlotExpression: + """ + Generate the sqlglot expression for an array of items with given pydough type. + + Args: + `items` : The list of SQLGlotExpressions representing the items in the array. + `inner_type` : The mapped PydDough type for the items in the array. + Returns: + A SQLGlotExpression representing the array of items. + """ + raise NotImplementedError( + f"Array types are not currently supported in dialect {self._visitor._expr_visitor._dialect.name}." + ) + def generate_dataframe_item_dialect_expression( self, item: Any, item_type: PyDoughType ) -> SQLGlotExpression: diff --git a/pydough/sqlglot/transform_bindings/mysql_transform_bindings.py b/pydough/sqlglot/transform_bindings/mysql_transform_bindings.py index c07625163..82692a0fb 100644 --- a/pydough/sqlglot/transform_bindings/mysql_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/mysql_transform_bindings.py @@ -93,6 +93,11 @@ def convert_listof( ) -> SQLGlotExpression: return sqlglot_expressions.Anonymous(this="JSON_ARRAYAGG", expressions=args) + def generate_dataframe_array_expression( + self, items: list[SQLGlotExpression], inner_type: PyDoughType + ) -> SQLGlotExpression: + return sqlglot_expressions.JSONArray(expressions=items) + def convert_slice( self, args: list[SQLGlotExpression], types: list[PyDoughType] ) -> SQLGlotExpression: diff --git a/pydough/sqlglot/transform_bindings/postgres_transform_bindings.py b/pydough/sqlglot/transform_bindings/postgres_transform_bindings.py index 62e5991cb..3e5c72086 100644 --- a/pydough/sqlglot/transform_bindings/postgres_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/postgres_transform_bindings.py @@ -71,6 +71,16 @@ def convert_listof( ) -> SQLGlotExpression: return sqlglot_expressions.Anonymous(this="ARRAY_AGG", expressions=args) + def generate_dataframe_array_expression( + self, items: list[SQLGlotExpression], inner_type: PyDoughType + ) -> SQLGlotExpression: + if len(items) == 0: + return sqlglot_expressions.Cast( + this=sqlglot_expressions.Array(expressions=[]), + to=sqlglot_expressions.DataType.build("TEXT[]", dialect="postgres"), + ) + return sqlglot_expressions.Array(expressions=items) + def convert_sum( self, arg: list[SQLGlotExpression], types: list[PyDoughType] ) -> SQLGlotExpression: diff --git a/pydough/sqlglot/transform_bindings/sf_transform_bindings.py b/pydough/sqlglot/transform_bindings/sf_transform_bindings.py index 269bb7d11..6e5b847a0 100644 --- a/pydough/sqlglot/transform_bindings/sf_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/sf_transform_bindings.py @@ -90,6 +90,11 @@ def convert_listof( ) -> SQLGlotExpression: return sqlglot_expressions.ArrayAgg(this=args[0]) + def generate_dataframe_array_expression( + self, items: list[SQLGlotExpression], inner_type: PyDoughType + ) -> SQLGlotExpression: + return sqlglot_expressions.Anonymous(this="ARRAY_CONSTRUCT", expressions=items) + def convert_integer( self, args: list[SQLGlotExpression], types: list[PyDoughType] ) -> SQLGlotExpression: diff --git a/pydough/sqlglot/transform_bindings/trino_transform_bindings.py b/pydough/sqlglot/transform_bindings/trino_transform_bindings.py index ef5764fa9..250187f82 100644 --- a/pydough/sqlglot/transform_bindings/trino_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/trino_transform_bindings.py @@ -131,6 +131,11 @@ def convert_listof( ) -> SQLGlotExpression: return sqlglot_expressions.ArrayAgg(this=args[0]) + def generate_dataframe_array_expression( + self, items: list[SQLGlotExpression], inner_type: PyDoughType + ) -> SQLGlotExpression: + return sqlglot_expressions.Array(expressions=items) + def convert_datediff( self, args: list[SQLGlotExpression], diff --git a/pydough/user_collections/dataframe_collection.py b/pydough/user_collections/dataframe_collection.py index 09e48d6d8..d37f17842 100644 --- a/pydough/user_collections/dataframe_collection.py +++ b/pydough/user_collections/dataframe_collection.py @@ -31,6 +31,7 @@ import pyarrow as pa import pyarrow.types as pa_types +from pydough.types.array_type import ArrayType from pydough.types.boolean_type import BooleanType from pydough.types.datetime_type import DatetimeType from pydough.types.numeric_type import NumericType @@ -205,17 +206,21 @@ def match_pyarrow_pydough_types( or pa.types.is_duration(field_type) ): return DatetimeType() + + elif pa_types.is_list(field_type) or pa.types.is_large_list(field_type): + inner_type: PyDoughType = ( + DataframeGeneratedCollection.match_pyarrow_pydough_types( + field_type.value_type, field_name + ) + ) + return ArrayType(inner_type) + # Unsupported types elif pa.types.is_binary(field_type) or pa.types.is_large_binary(field_type): raise ValueError( f"Binaries in column '{field_name}', are not supported for dataframe collections" ) - elif pa_types.is_list(field_type) or pa.types.is_large_list(field_type): - raise ValueError( - f"Arrays in column '{field_name}', are not supported for dataframe collections" - ) - elif pa_types.is_struct(field_type): raise ValueError( f"Structs in column '{field_name}', are not supported for dataframe collections" diff --git a/tests/test_pipeline_tpch_custom.py b/tests/test_pipeline_tpch_custom.py index 2c99007f2..5976cacf3 100644 --- a/tests/test_pipeline_tpch_custom.py +++ b/tests/test_pipeline_tpch_custom.py @@ -7,7 +7,7 @@ import logging import re from collections.abc import Callable -from datetime import date +from datetime import date, datetime from decimal import Decimal import numpy as np @@ -4112,10 +4112,137 @@ ), id="array_data_01", ), + pytest.param( + PyDoughPandasTest( + "result = pydough.dataframe_collection(name='tbl', dataframe=array_df, unique_column_names=['idx'])", + "TPCH", + lambda: pd.DataFrame( + { + "idx": [1, 2, 3, 4], + "arr_s": [["A"], [], ["B", "C"], ["D", "E", None, "F"]], + "arr_i": [[10], [], [20, 30], [40, 50, None, 60]], + } + ), + "array_data_02", + order_sensitive=True, + skipped_dialects={"ANSI", "SQLITE"}, + kwargs={ + "array_df": pd.DataFrame( + { + "idx": [1, 2, 3, 4], + "arr_s": [["A"], [], ["B", "C"], ["D", "E", None, "F"]], + "arr_i": [[10], [], [20, 30], [40, 50, None, 60]], + } + ) + }, + ), + id="array_data_02", + ), + pytest.param( + PyDoughPandasTest( + "result = pydough.dataframe_collection(name='tbl', dataframe=array_df, unique_column_names=['idx'])", + "TPCH", + lambda: pd.DataFrame( + { + "idx": [1, 2, 3, 4], + "arr_f": [ + [1.1], + [], + [-2.3, 0.0], + [3.14, None, float("nan"), float("inf"), float("-inf")], + ], + } + ), + "array_data_02", + order_sensitive=True, + skipped_dialects={"ANSI", "SQLITE", "MYSQL"}, + kwargs={ + "array_df": pd.DataFrame( + { + "idx": [1, 2, 3, 4], + "arr_f": [ + [1.1], + [], + [-2.3, 0.0], + [3.14, None, float("nan"), float("inf"), float("-inf")], + ], + } + ) + }, + ), + id="array_data_03", + ), + pytest.param( + PyDoughPandasTest( + "result = pydough.dataframe_collection(name='tbl', dataframe=array_df, unique_column_names=['idx'])", + "TPCH", + lambda: pd.DataFrame( + { + "idx": [1, 2, 3, 4], + "arr_d": [ + [datetime(2020, 1, 1, 0, 0, 0)], + [], + [ + datetime(2021, 6, 15, 0, 0, 0), + datetime(2022, 12, 31, 0, 0, 0), + ], + [ + datetime(1999, 7, 4, 0, 0, 0), + None, + datetime(2000, 1, 1, 0, 0, 0), + ], + ], + "arr_t": [ + [pd.Timestamp("2020-01-01 12:00:00")], + [], + [pd.Timestamp("2021-06-15"), pd.Timestamp("2022-12-31")], + [ + pd.Timestamp("1999-07-04 23:15:00"), + None, + pd.Timestamp("2000-01-01"), + ], + ], + } + ), + "array_data_04", + order_sensitive=True, + skipped_dialects={"ANSI", "SQLITE"}, + kwargs={ + "array_df": pd.DataFrame( + { + "idx": [1, 2, 3, 4], + "arr_d": [ + [date(2020, 1, 1)], + [], + [date(2021, 6, 15), date(2022, 12, 31)], + [date(1999, 7, 4), None, date(2000, 1, 1)], + ], + "arr_t": [ + [pd.Timestamp("2020-01-01 12:00:00")], + [], + [ + pd.Timestamp("2021-06-15"), + pd.Timestamp("2022-12-31"), + ], + [ + pd.Timestamp("1999-07-04 23:15:00"), + None, + pd.Timestamp("2000-01-01"), + ], + ], + } + ) + }, + ), + id="array_data_04", + ), pytest.param( PyDoughPandasTest( "array_data = regions.CALCULATE(nation_names=LISTOF(nations.name))\n" - "result = array_data.EXPLODE(nation_names, index_name='nation_idx', value_name='nation_name').ORDER_BY(region_name, nation_idx)", + "result = (" + " array_data" + " .EXPLODE(nation_names, index_name='nation_idx', value_name='nation_name', keep=True, filtering=False, is_distinct=True)" + " .ORDER_BY(region_name, nation_idx)", "TPCH", lambda: pd.DataFrame( { @@ -4124,12 +4251,39 @@ + ["ASIA"] * 5 + ["EUROPE"] * 5 + ["MIDDLE EAST"] * 5, + "nation_names": [ + [["ALGERIA", "ETHIOPIA", "KENYA", "MOROCCO", "MOZAMBIQUE"]] + * 5 + + [ + [ + "ARGENTINA", + "BRAZIL", + "CANADA", + "PERU", + "UNITED STATES", + ] + ] + * 5 + + [["INDIA", "INDONESIA", "JAPAN", "CHINA", "VIETNAM"]] * 5 + + [ + [ + "FRANCE", + "GERMANY", + "ROMANIA", + "RUSSIA", + "UNITED KINGDOM", + ] + ] + * 5 + + [["EGYPT", "IRAN", "IRAQ", "JORDAN", "SAUDI ARABIA"]] * 5, + ], "nation_idx": list(range(5)) * 5, "nation_name": [ "ALGERIA", "ETHIOPIA", "KENYA", "MOROCCO", + "MOROCCO", "MOZAMBIQUE", "ARGENTINA", "BRAZIL", diff --git a/tests/test_plan_refsols/array_data_02.txt b/tests/test_plan_refsols/array_data_02.txt new file mode 100644 index 000000000..22602d842 --- /dev/null +++ b/tests/test_plan_refsols/array_data_02.txt @@ -0,0 +1,2 @@ +ROOT(columns=[('idx', idx), ('arr_f', arr_f)], orderings=[]) + GENERATED_TABLE(DataframeCollection(name='tbl', shape=(4, 2), columns=['idx', 'arr_f'])) diff --git a/tests/test_plan_refsols/array_data_04.txt b/tests/test_plan_refsols/array_data_04.txt new file mode 100644 index 000000000..58443e042 --- /dev/null +++ b/tests/test_plan_refsols/array_data_04.txt @@ -0,0 +1,2 @@ +ROOT(columns=[('idx', idx), ('arr_d', arr_d), ('arr_t', arr_t)], orderings=[]) + GENERATED_TABLE(DataframeCollection(name='tbl', shape=(4, 3), columns=['idx', 'arr_d', 'arr_t'])) diff --git a/tests/test_sql_refsols/array_data_02_mysql.sql b/tests/test_sql_refsols/array_data_02_mysql.sql new file mode 100644 index 000000000..b182afa21 --- /dev/null +++ b/tests/test_sql_refsols/array_data_02_mysql.sql @@ -0,0 +1,9 @@ +SELECT + tbl.idx, + tbl.arr_s, + tbl.arr_i +FROM (VALUES + ROW(1, JSON_ARRAY('A'), JSON_ARRAY(10)), + ROW(2, JSON_ARRAY(), JSON_ARRAY()), + ROW(3, JSON_ARRAY('B', 'C'), JSON_ARRAY(20, 30)), + ROW(4, JSON_ARRAY('D', 'E', NULL, 'F'), JSON_ARRAY(40, 50, NULL, 60))) AS tbl(idx, arr_s, arr_i) diff --git a/tests/test_sql_refsols/array_data_02_postgres.sql b/tests/test_sql_refsols/array_data_02_postgres.sql new file mode 100644 index 000000000..7af080872 --- /dev/null +++ b/tests/test_sql_refsols/array_data_02_postgres.sql @@ -0,0 +1,8 @@ +SELECT + tbl.idx, + tbl.arr_f +FROM (VALUES + (1, ARRAY[1.1]), + (2, ARRAY[]), + (3, ARRAY[-2.3, 0.0]), + (4, ARRAY[3.14, NULL, NULL, CAST('inf' AS REAL), CAST('-inf' AS REAL)])) AS tbl(idx, arr_f) diff --git a/tests/test_sql_refsols/array_data_02_snowflake.sql b/tests/test_sql_refsols/array_data_02_snowflake.sql new file mode 100644 index 000000000..0b009b621 --- /dev/null +++ b/tests/test_sql_refsols/array_data_02_snowflake.sql @@ -0,0 +1,8 @@ +SELECT + tbl.idx, + tbl.arr_f +FROM (VALUES + (1, [1.1]), + (2, []), + (3, [-2.3, 0.0]), + (4, [3.14, NULL, NULL, TO_DOUBLE('INF'), TO_DOUBLE('-INF')])) AS tbl(idx, arr_f) diff --git a/tests/test_sql_refsols/array_data_02_trino.sql b/tests/test_sql_refsols/array_data_02_trino.sql new file mode 100644 index 000000000..99ca22eb3 --- /dev/null +++ b/tests/test_sql_refsols/array_data_02_trino.sql @@ -0,0 +1,11 @@ +SELECT + tbl.idx, + tbl.arr_f +FROM (VALUES + (1, ARRAY[1.1]), + (2, ARRAY[]), + (3, ARRAY[-2.3, 0.0]), + ( + 4, + ARRAY[3.14, NULL, NULL, CAST('Infinity' AS DOUBLE), CAST('-Infinity' AS DOUBLE)] + )) AS tbl(idx, arr_f) diff --git a/tests/test_sql_refsols/array_data_04_mysql.sql b/tests/test_sql_refsols/array_data_04_mysql.sql new file mode 100644 index 000000000..6763ca303 --- /dev/null +++ b/tests/test_sql_refsols/array_data_04_mysql.sql @@ -0,0 +1,25 @@ +SELECT + tbl.idx, + tbl.arr_d, + tbl.arr_t +FROM (VALUES + ROW( + 1, + JSON_ARRAY(CAST('2020-01-01' AS DATETIME)), + JSON_ARRAY(CAST('2020-01-01 12:00:00' AS DATETIME)) + ), + ROW(2, JSON_ARRAY(), JSON_ARRAY()), + ROW( + 3, + JSON_ARRAY(CAST('2021-06-15' AS DATETIME), CAST('2022-12-31' AS DATETIME)), + JSON_ARRAY(CAST('2021-06-15 00:00:00' AS DATETIME), CAST('2022-12-31 00:00:00' AS DATETIME)) + ), + ROW( + 4, + JSON_ARRAY(CAST('1999-07-04' AS DATETIME), NULL, CAST('2000-01-01' AS DATETIME)), + JSON_ARRAY( + CAST('1999-07-04 23:15:00' AS DATETIME), + NULL, + CAST('2000-01-01 00:00:00' AS DATETIME) + ) + )) AS tbl(idx, arr_d, arr_t) diff --git a/tests/test_sql_refsols/array_data_04_postgres.sql b/tests/test_sql_refsols/array_data_04_postgres.sql new file mode 100644 index 000000000..0e08f6a97 --- /dev/null +++ b/tests/test_sql_refsols/array_data_04_postgres.sql @@ -0,0 +1,21 @@ +SELECT + tbl.idx, + tbl.arr_d, + tbl.arr_t +FROM (VALUES + ( + 1, + ARRAY[CAST('2020-01-01' AS TIMESTAMP)], + ARRAY[CAST('2020-01-01 12:00:00' AS TIMESTAMP)] + ), + (2, ARRAY[], ARRAY[]), + ( + 3, + ARRAY[CAST('2021-06-15' AS TIMESTAMP), CAST('2022-12-31' AS TIMESTAMP)], + ARRAY[CAST('2021-06-15 00:00:00' AS TIMESTAMP), CAST('2022-12-31 00:00:00' AS TIMESTAMP)] + ), + ( + 4, + ARRAY[CAST('1999-07-04' AS TIMESTAMP), NULL, CAST('2000-01-01' AS TIMESTAMP)], + ARRAY[CAST('1999-07-04 23:15:00' AS TIMESTAMP), NULL, CAST('2000-01-01 00:00:00' AS TIMESTAMP)] + )) AS tbl(idx, arr_d, arr_t) diff --git a/tests/test_sql_refsols/array_data_04_snowflake.sql b/tests/test_sql_refsols/array_data_04_snowflake.sql new file mode 100644 index 000000000..7067125a6 --- /dev/null +++ b/tests/test_sql_refsols/array_data_04_snowflake.sql @@ -0,0 +1,24 @@ +SELECT + tbl.idx, + tbl.arr_d, + tbl.arr_t +FROM (VALUES + (1, [CAST('2020-01-01' AS TIMESTAMP)], [CAST('2020-01-01 12:00:00' AS TIMESTAMP)]), + (2, [], []), + ( + 3, + [CAST('2021-06-15' AS TIMESTAMP), CAST('2022-12-31' AS TIMESTAMP)], + [ + CAST('2021-06-15 00:00:00' AS TIMESTAMP), + CAST('2022-12-31 00:00:00' AS TIMESTAMP) + ] + ), + ( + 4, + [CAST('1999-07-04' AS TIMESTAMP), NULL, CAST('2000-01-01' AS TIMESTAMP)], + [ + CAST('1999-07-04 23:15:00' AS TIMESTAMP), + NULL, + CAST('2000-01-01 00:00:00' AS TIMESTAMP) + ] + )) AS tbl(idx, arr_d, arr_t) diff --git a/tests/test_sql_refsols/array_data_04_trino.sql b/tests/test_sql_refsols/array_data_04_trino.sql new file mode 100644 index 000000000..0e08f6a97 --- /dev/null +++ b/tests/test_sql_refsols/array_data_04_trino.sql @@ -0,0 +1,21 @@ +SELECT + tbl.idx, + tbl.arr_d, + tbl.arr_t +FROM (VALUES + ( + 1, + ARRAY[CAST('2020-01-01' AS TIMESTAMP)], + ARRAY[CAST('2020-01-01 12:00:00' AS TIMESTAMP)] + ), + (2, ARRAY[], ARRAY[]), + ( + 3, + ARRAY[CAST('2021-06-15' AS TIMESTAMP), CAST('2022-12-31' AS TIMESTAMP)], + ARRAY[CAST('2021-06-15 00:00:00' AS TIMESTAMP), CAST('2022-12-31 00:00:00' AS TIMESTAMP)] + ), + ( + 4, + ARRAY[CAST('1999-07-04' AS TIMESTAMP), NULL, CAST('2000-01-01' AS TIMESTAMP)], + ARRAY[CAST('1999-07-04 23:15:00' AS TIMESTAMP), NULL, CAST('2000-01-01 00:00:00' AS TIMESTAMP)] + )) AS tbl(idx, arr_d, arr_t) From 4ecdaf4b90a5650fa8c9c3d4d93eb38eb81dea4e Mon Sep 17 00:00:00 2001 From: knassre-bodo Date: Tue, 2 Jun 2026 13:56:14 -0700 Subject: [PATCH 08/44] Array literal dialect WIP --- .../postgres_transform_bindings.py | 24 +++++++++++++++---- .../array_data_02_postgres.sql | 4 +++- .../array_data_04_postgres.sql | 10 +++++++- 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/pydough/sqlglot/transform_bindings/postgres_transform_bindings.py b/pydough/sqlglot/transform_bindings/postgres_transform_bindings.py index 3e5c72086..54b846452 100644 --- a/pydough/sqlglot/transform_bindings/postgres_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/postgres_transform_bindings.py @@ -8,6 +8,7 @@ from typing import Any import sqlglot.expressions as sqlglot_expressions +from sqlglot import parse_one from sqlglot.expressions import Expression as SQLGlotExpression import pydough.pydough_operators as pydop @@ -15,7 +16,7 @@ LiteralExpression, ) from pydough.sqlglot.sqlglot_helpers import normalize_column_name -from pydough.types import PyDoughType +from pydough.types import PyDoughType, StringType from pydough.types.boolean_type import BooleanType from pydough.types.datetime_type import DatetimeType from pydough.types.numeric_type import NumericType @@ -75,10 +76,23 @@ def generate_dataframe_array_expression( self, items: list[SQLGlotExpression], inner_type: PyDoughType ) -> SQLGlotExpression: if len(items) == 0: - return sqlglot_expressions.Cast( - this=sqlglot_expressions.Array(expressions=[]), - to=sqlglot_expressions.DataType.build("TEXT[]", dialect="postgres"), - ) + # Convoluted way to generate empty array literal in Postgres using + # SQLGlot, to work around Postgres's typing rules. + inner_term: str + match inner_type: + case BooleanType(): + inner_term = "true" + case StringType(): + inner_term = "''" + case NumericType(): + inner_term = "0" + case DatetimeType(): + inner_term = "CAST('1970-01-01' AS TIMESTAMP)" + case _: + raise ValueError( + f"Cannot support empty array of type {inner_type} in Postgres." + ) + return parse_one(f"(ARRAY[{inner_term}])[1:0]") return sqlglot_expressions.Array(expressions=items) def convert_sum( diff --git a/tests/test_sql_refsols/array_data_02_postgres.sql b/tests/test_sql_refsols/array_data_02_postgres.sql index 7af080872..4f15a7f3d 100644 --- a/tests/test_sql_refsols/array_data_02_postgres.sql +++ b/tests/test_sql_refsols/array_data_02_postgres.sql @@ -3,6 +3,8 @@ SELECT tbl.arr_f FROM (VALUES (1, ARRAY[1.1]), - (2, ARRAY[]), + (2, ( + ARRAY[0] + )[1 : 0]), (3, ARRAY[-2.3, 0.0]), (4, ARRAY[3.14, NULL, NULL, CAST('inf' AS REAL), CAST('-inf' AS REAL)])) AS tbl(idx, arr_f) diff --git a/tests/test_sql_refsols/array_data_04_postgres.sql b/tests/test_sql_refsols/array_data_04_postgres.sql index 0e08f6a97..47bc68a8d 100644 --- a/tests/test_sql_refsols/array_data_04_postgres.sql +++ b/tests/test_sql_refsols/array_data_04_postgres.sql @@ -8,7 +8,15 @@ FROM (VALUES ARRAY[CAST('2020-01-01' AS TIMESTAMP)], ARRAY[CAST('2020-01-01 12:00:00' AS TIMESTAMP)] ), - (2, ARRAY[], ARRAY[]), + ( + 2, + ( + ARRAY[CAST('1970-01-01' AS TIMESTAMP)] + )[1 : 0], + ( + ARRAY[CAST('1970-01-01' AS TIMESTAMP)] + )[1 : 0] + ), ( 3, ARRAY[CAST('2021-06-15' AS TIMESTAMP), CAST('2022-12-31' AS TIMESTAMP)], From f6f08774e2ea93c4f462fe9a29178c956292c002 Mon Sep 17 00:00:00 2001 From: knassre-bodo Date: Thu, 4 Jun 2026 10:18:43 -0700 Subject: [PATCH 09/44] Array data tests handled except for ORACLE --- .../transform_bindings/sf_transform_bindings.py | 5 ----- tests/test_pipeline_tpch_custom.py | 6 +++--- tests/testing_utilities.py | 17 +++++++++++++++++ 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/pydough/sqlglot/transform_bindings/sf_transform_bindings.py b/pydough/sqlglot/transform_bindings/sf_transform_bindings.py index 6e5b847a0..269bb7d11 100644 --- a/pydough/sqlglot/transform_bindings/sf_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/sf_transform_bindings.py @@ -90,11 +90,6 @@ def convert_listof( ) -> SQLGlotExpression: return sqlglot_expressions.ArrayAgg(this=args[0]) - def generate_dataframe_array_expression( - self, items: list[SQLGlotExpression], inner_type: PyDoughType - ) -> SQLGlotExpression: - return sqlglot_expressions.Anonymous(this="ARRAY_CONSTRUCT", expressions=items) - def convert_integer( self, args: list[SQLGlotExpression], types: list[PyDoughType] ) -> SQLGlotExpression: diff --git a/tests/test_pipeline_tpch_custom.py b/tests/test_pipeline_tpch_custom.py index 5976cacf3..356a32136 100644 --- a/tests/test_pipeline_tpch_custom.py +++ b/tests/test_pipeline_tpch_custom.py @@ -4125,7 +4125,7 @@ ), "array_data_02", order_sensitive=True, - skipped_dialects={"ANSI", "SQLITE"}, + skipped_dialects={"ANSI", "SQLITE", "SNOWFLAKE"}, kwargs={ "array_df": pd.DataFrame( { @@ -4155,7 +4155,7 @@ ), "array_data_02", order_sensitive=True, - skipped_dialects={"ANSI", "SQLITE", "MYSQL"}, + skipped_dialects={"ANSI", "SQLITE", "MYSQL", "SNOWFLAKE"}, kwargs={ "array_df": pd.DataFrame( { @@ -4206,7 +4206,7 @@ ), "array_data_04", order_sensitive=True, - skipped_dialects={"ANSI", "SQLITE"}, + skipped_dialects={"ANSI", "SQLITE", "SNOWFLAKE"}, kwargs={ "array_df": pd.DataFrame( { diff --git a/tests/testing_utilities.py b/tests/testing_utilities.py index 8c6d71b2a..a4a680271 100644 --- a/tests/testing_utilities.py +++ b/tests/testing_utilities.py @@ -1853,6 +1853,23 @@ def sanitize_string(val): ): return column_a, column_b.apply(lambda x: pd.NA if pd.isna(x) else x.date()) + # For array types, harmonize the inner arrays + if ( + pd.api.types.is_object_dtype(column_a) + and pd.api.types.is_object_dtype(column_b) + and any(isinstance(col, list) for col in column_a) + and any(isinstance(col, list) for col in column_b) + ): + for i in range(len(column_a)): + if isinstance(column_a[i], list) and isinstance(column_b[i], list): + column_a[i], column_b[i] = harmonize_types( + pd.Series(column_a[i]), pd.Series(column_b[i]) + ) + # After harmonizing types, convert back to list for comparison, + # but ensure that NAT is converted to None. + column_a[i] = [None if pd.isna(x) else x for x in column_a[i].tolist()] + column_b[i] = [None if pd.isna(x) else x for x in column_b[i].tolist()] + return column_a, column_b From 99098a00d694233bc106ea51ef39528f13478685 Mon Sep 17 00:00:00 2001 From: knassre-bodo Date: Thu, 11 Jun 2026 09:26:04 -0700 Subject: [PATCH 10/44] Testing WIP --- .../oracle_transform_bindings.py | 17 +++++++++ tests/test_pipeline_tpch_custom.py | 2 +- tests/test_plan_refsols/array_data_02.txt | 4 +-- tests/test_plan_refsols/array_data_03.txt | 2 ++ .../test_sql_refsols/array_data_02_oracle.sql | 13 +++++++ .../array_data_02_postgres.sql | 11 +++--- .../array_data_02_snowflake.sql | 8 ----- .../test_sql_refsols/array_data_02_trino.sql | 14 ++++---- .../test_sql_refsols/array_data_03_oracle.sql | 11 ++++++ .../array_data_03_postgres.sql | 10 ++++++ .../test_sql_refsols/array_data_03_trino.sql | 11 ++++++ .../test_sql_refsols/array_data_04_oracle.sql | 35 +++++++++++++++++++ .../array_data_04_snowflake.sql | 24 ------------- 13 files changed, 115 insertions(+), 47 deletions(-) create mode 100644 tests/test_plan_refsols/array_data_03.txt create mode 100644 tests/test_sql_refsols/array_data_02_oracle.sql delete mode 100644 tests/test_sql_refsols/array_data_02_snowflake.sql create mode 100644 tests/test_sql_refsols/array_data_03_oracle.sql create mode 100644 tests/test_sql_refsols/array_data_03_postgres.sql create mode 100644 tests/test_sql_refsols/array_data_03_trino.sql create mode 100644 tests/test_sql_refsols/array_data_04_oracle.sql delete mode 100644 tests/test_sql_refsols/array_data_04_snowflake.sql diff --git a/pydough/sqlglot/transform_bindings/oracle_transform_bindings.py b/pydough/sqlglot/transform_bindings/oracle_transform_bindings.py index 4afeaa1f2..7e75aa6ca 100644 --- a/pydough/sqlglot/transform_bindings/oracle_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/oracle_transform_bindings.py @@ -102,6 +102,23 @@ def convert_listof( ) -> SQLGlotExpression: return sqlglot_expressions.Anonymous(this="JSON_ARRAYAGG", expressions=args) + def generate_dataframe_array_expression( + self, items: list[SQLGlotExpression], inner_type: PyDoughType + ) -> SQLGlotExpression: + func: str + match inner_type: + case StringType(): + func = "SYS.ODCIVARCHAR2LIST" + case NumericType(): + func = "SYS.ODCINUMBERLIST" + case DatetimeType(): + func = "SYS.ODCIDATELIST" + case _: + raise ValueError( + f"Cannot support constant array of type {inner_type} in Oracle." + ) + return sqlglot_expressions.Anonymous(this=func, expressions=items) + def convert_default_to( self, args: list[SQLGlotExpression], types: list[PyDoughType] ) -> SQLGlotExpression: diff --git a/tests/test_pipeline_tpch_custom.py b/tests/test_pipeline_tpch_custom.py index 356a32136..efdbcb9de 100644 --- a/tests/test_pipeline_tpch_custom.py +++ b/tests/test_pipeline_tpch_custom.py @@ -4153,7 +4153,7 @@ ], } ), - "array_data_02", + "array_data_03", order_sensitive=True, skipped_dialects={"ANSI", "SQLITE", "MYSQL", "SNOWFLAKE"}, kwargs={ diff --git a/tests/test_plan_refsols/array_data_02.txt b/tests/test_plan_refsols/array_data_02.txt index 22602d842..4890c04fd 100644 --- a/tests/test_plan_refsols/array_data_02.txt +++ b/tests/test_plan_refsols/array_data_02.txt @@ -1,2 +1,2 @@ -ROOT(columns=[('idx', idx), ('arr_f', arr_f)], orderings=[]) - GENERATED_TABLE(DataframeCollection(name='tbl', shape=(4, 2), columns=['idx', 'arr_f'])) +ROOT(columns=[('idx', idx), ('arr_s', arr_s), ('arr_i', arr_i)], orderings=[]) + GENERATED_TABLE(DataframeCollection(name='tbl', shape=(4, 3), columns=['idx', 'arr_s', 'arr_i'])) diff --git a/tests/test_plan_refsols/array_data_03.txt b/tests/test_plan_refsols/array_data_03.txt new file mode 100644 index 000000000..22602d842 --- /dev/null +++ b/tests/test_plan_refsols/array_data_03.txt @@ -0,0 +1,2 @@ +ROOT(columns=[('idx', idx), ('arr_f', arr_f)], orderings=[]) + GENERATED_TABLE(DataframeCollection(name='tbl', shape=(4, 2), columns=['idx', 'arr_f'])) diff --git a/tests/test_sql_refsols/array_data_02_oracle.sql b/tests/test_sql_refsols/array_data_02_oracle.sql new file mode 100644 index 000000000..4c755b3d6 --- /dev/null +++ b/tests/test_sql_refsols/array_data_02_oracle.sql @@ -0,0 +1,13 @@ +SELECT + TBL.IDX AS idx, + TBL.ARR_S AS arr_s, + TBL.ARR_I AS arr_i +FROM (VALUES + (1, SYS.ODCIVARCHAR2LIST('A'), SYS.ODCINUMBERLIST(10)), + (2, SYS.ODCIVARCHAR2LIST(), SYS.ODCINUMBERLIST()), + (3, SYS.ODCIVARCHAR2LIST('B', 'C'), SYS.ODCINUMBERLIST(20, 30)), + ( + 4, + SYS.ODCIVARCHAR2LIST('D', 'E', NULL, 'F'), + SYS.ODCINUMBERLIST(40, 50, NULL, 60) + )) AS TBL(IDX, ARR_S, ARR_I) diff --git a/tests/test_sql_refsols/array_data_02_postgres.sql b/tests/test_sql_refsols/array_data_02_postgres.sql index 4f15a7f3d..c13582772 100644 --- a/tests/test_sql_refsols/array_data_02_postgres.sql +++ b/tests/test_sql_refsols/array_data_02_postgres.sql @@ -1,10 +1,13 @@ SELECT tbl.idx, - tbl.arr_f + tbl.arr_s, + tbl.arr_i FROM (VALUES - (1, ARRAY[1.1]), + (1, ARRAY['A'], ARRAY[10]), (2, ( + ARRAY[''] + )[1 : 0], ( ARRAY[0] )[1 : 0]), - (3, ARRAY[-2.3, 0.0]), - (4, ARRAY[3.14, NULL, NULL, CAST('inf' AS REAL), CAST('-inf' AS REAL)])) AS tbl(idx, arr_f) + (3, ARRAY['B', 'C'], ARRAY[20, 30]), + (4, ARRAY['D', 'E', NULL, 'F'], ARRAY[40, 50, NULL, 60])) AS tbl(idx, arr_s, arr_i) diff --git a/tests/test_sql_refsols/array_data_02_snowflake.sql b/tests/test_sql_refsols/array_data_02_snowflake.sql deleted file mode 100644 index 0b009b621..000000000 --- a/tests/test_sql_refsols/array_data_02_snowflake.sql +++ /dev/null @@ -1,8 +0,0 @@ -SELECT - tbl.idx, - tbl.arr_f -FROM (VALUES - (1, [1.1]), - (2, []), - (3, [-2.3, 0.0]), - (4, [3.14, NULL, NULL, TO_DOUBLE('INF'), TO_DOUBLE('-INF')])) AS tbl(idx, arr_f) diff --git a/tests/test_sql_refsols/array_data_02_trino.sql b/tests/test_sql_refsols/array_data_02_trino.sql index 99ca22eb3..84d71d955 100644 --- a/tests/test_sql_refsols/array_data_02_trino.sql +++ b/tests/test_sql_refsols/array_data_02_trino.sql @@ -1,11 +1,9 @@ SELECT tbl.idx, - tbl.arr_f + tbl.arr_s, + tbl.arr_i FROM (VALUES - (1, ARRAY[1.1]), - (2, ARRAY[]), - (3, ARRAY[-2.3, 0.0]), - ( - 4, - ARRAY[3.14, NULL, NULL, CAST('Infinity' AS DOUBLE), CAST('-Infinity' AS DOUBLE)] - )) AS tbl(idx, arr_f) + (1, ARRAY['A'], ARRAY[10]), + (2, ARRAY[], ARRAY[]), + (3, ARRAY['B', 'C'], ARRAY[20, 30]), + (4, ARRAY['D', 'E', NULL, 'F'], ARRAY[40, 50, NULL, 60])) AS tbl(idx, arr_s, arr_i) diff --git a/tests/test_sql_refsols/array_data_03_oracle.sql b/tests/test_sql_refsols/array_data_03_oracle.sql new file mode 100644 index 000000000..d46743970 --- /dev/null +++ b/tests/test_sql_refsols/array_data_03_oracle.sql @@ -0,0 +1,11 @@ +SELECT + TBL.IDX AS idx, + TBL.ARR_F AS arr_f +FROM (VALUES + (1, SYS.ODCINUMBERLIST(1.1)), + (2, SYS.ODCINUMBERLIST()), + (3, SYS.ODCINUMBERLIST(-2.3, 0.0)), + ( + 4, + SYS.ODCINUMBERLIST(3.14, NULL, NULL, BINARY_DOUBLE_INFINITY, -BINARY_DOUBLE_INFINITY) + )) AS TBL(IDX, ARR_F) diff --git a/tests/test_sql_refsols/array_data_03_postgres.sql b/tests/test_sql_refsols/array_data_03_postgres.sql new file mode 100644 index 000000000..4f15a7f3d --- /dev/null +++ b/tests/test_sql_refsols/array_data_03_postgres.sql @@ -0,0 +1,10 @@ +SELECT + tbl.idx, + tbl.arr_f +FROM (VALUES + (1, ARRAY[1.1]), + (2, ( + ARRAY[0] + )[1 : 0]), + (3, ARRAY[-2.3, 0.0]), + (4, ARRAY[3.14, NULL, NULL, CAST('inf' AS REAL), CAST('-inf' AS REAL)])) AS tbl(idx, arr_f) diff --git a/tests/test_sql_refsols/array_data_03_trino.sql b/tests/test_sql_refsols/array_data_03_trino.sql new file mode 100644 index 000000000..99ca22eb3 --- /dev/null +++ b/tests/test_sql_refsols/array_data_03_trino.sql @@ -0,0 +1,11 @@ +SELECT + tbl.idx, + tbl.arr_f +FROM (VALUES + (1, ARRAY[1.1]), + (2, ARRAY[]), + (3, ARRAY[-2.3, 0.0]), + ( + 4, + ARRAY[3.14, NULL, NULL, CAST('Infinity' AS DOUBLE), CAST('-Infinity' AS DOUBLE)] + )) AS tbl(idx, arr_f) diff --git a/tests/test_sql_refsols/array_data_04_oracle.sql b/tests/test_sql_refsols/array_data_04_oracle.sql new file mode 100644 index 000000000..d63532055 --- /dev/null +++ b/tests/test_sql_refsols/array_data_04_oracle.sql @@ -0,0 +1,35 @@ +SELECT + TBL.IDX AS idx, + TBL.ARR_D AS arr_d, + TBL.ARR_T AS arr_t +FROM (VALUES + ( + 1, + SYS.ODCIDATELIST(TO_DATE('2020-01-01', 'YYYY-MM-DD HH24:MI:SS')), + SYS.ODCIDATELIST(TO_DATE('2020-01-01 12:00:00', 'YYYY-MM-DD HH24:MI:SS')) + ), + (2, SYS.ODCIDATELIST(), SYS.ODCIDATELIST()), + ( + 3, + SYS.ODCIDATELIST( + TO_DATE('2021-06-15', 'YYYY-MM-DD HH24:MI:SS'), + TO_DATE('2022-12-31', 'YYYY-MM-DD HH24:MI:SS') + ), + SYS.ODCIDATELIST( + TO_DATE('2021-06-15 00:00:00', 'YYYY-MM-DD HH24:MI:SS'), + TO_DATE('2022-12-31 00:00:00', 'YYYY-MM-DD HH24:MI:SS') + ) + ), + ( + 4, + SYS.ODCIDATELIST( + TO_DATE('1999-07-04', 'YYYY-MM-DD HH24:MI:SS'), + NULL, + TO_DATE('2000-01-01', 'YYYY-MM-DD HH24:MI:SS') + ), + SYS.ODCIDATELIST( + TO_DATE('1999-07-04 23:15:00', 'YYYY-MM-DD HH24:MI:SS'), + NULL, + TO_DATE('2000-01-01 00:00:00', 'YYYY-MM-DD HH24:MI:SS') + ) + )) AS TBL(IDX, ARR_D, ARR_T) diff --git a/tests/test_sql_refsols/array_data_04_snowflake.sql b/tests/test_sql_refsols/array_data_04_snowflake.sql deleted file mode 100644 index 7067125a6..000000000 --- a/tests/test_sql_refsols/array_data_04_snowflake.sql +++ /dev/null @@ -1,24 +0,0 @@ -SELECT - tbl.idx, - tbl.arr_d, - tbl.arr_t -FROM (VALUES - (1, [CAST('2020-01-01' AS TIMESTAMP)], [CAST('2020-01-01 12:00:00' AS TIMESTAMP)]), - (2, [], []), - ( - 3, - [CAST('2021-06-15' AS TIMESTAMP), CAST('2022-12-31' AS TIMESTAMP)], - [ - CAST('2021-06-15 00:00:00' AS TIMESTAMP), - CAST('2022-12-31 00:00:00' AS TIMESTAMP) - ] - ), - ( - 4, - [CAST('1999-07-04' AS TIMESTAMP), NULL, CAST('2000-01-01' AS TIMESTAMP)], - [ - CAST('1999-07-04 23:15:00' AS TIMESTAMP), - NULL, - CAST('2000-01-01 00:00:00' AS TIMESTAMP) - ] - )) AS tbl(idx, arr_d, arr_t) From 4120c9b97d84ca3d1681bc92549ab51a254de4e6 Mon Sep 17 00:00:00 2001 From: knassre-bodo Date: Thu, 11 Jun 2026 09:56:27 -0700 Subject: [PATCH 11/44] Test run [RUN ORACLE] From df06954e7cdfb670652845140b57ef50b902c169 Mon Sep 17 00:00:00 2001 From: knassre-bodo Date: Thu, 11 Jun 2026 10:06:01 -0700 Subject: [PATCH 12/44] Test run oracle 2 [RUN ORACLE] From 3be7bd35c75fc8ed5942f0fde796a10a24684d29 Mon Sep 17 00:00:00 2001 From: knassre-bodo Date: Wed, 17 Jun 2026 16:27:32 -0700 Subject: [PATCH 13/44] Early testing and unqualified nodes WIP --- pydough/unqualified/unqualified_node.py | 131 +++++++++++++++++++++++- tests/test_pipeline_tpch_custom.py | 46 ++++++++- tests/test_unqualified_node.py | 10 ++ 3 files changed, 180 insertions(+), 7 deletions(-) diff --git a/pydough/unqualified/unqualified_node.py b/pydough/unqualified/unqualified_node.py index c0f693cb6..39fdb39d5 100644 --- a/pydough/unqualified/unqualified_node.py +++ b/pydough/unqualified/unqualified_node.py @@ -435,6 +435,81 @@ def BEST( return UnqualifiedBest(self, by, per, allow_ties, n_best) + def EXPLODE( + self, + data: "UnqualifiedNode", + value_name: str, + index_name: str | None = None, + version: str = "array", + delimiter: str | None = None, + keep: bool = False, + filtering: bool = True, + is_distinct: bool = False, + ): + """ + Method used to create an EXPLODE node, transforming the existing + collection by exploding each row into multiple rows based on the values + in an array column, or from splitting a string column on a delimiter. + + Args: + `data`: the array or string column to explode. + `value_name`: the name of the column representing the exploded + values. + `index_name` (optional): the name of the column representing the + index of each exploded value within its original array or string. + If None, no such column will be created. This must be provided when + `is_distinct` is False. Default is None. + `version` (optional): the version of explode to use. Must be one of "array" + or "string". "array" should be used when exploding an array column, + and "string" should be used when exploding a string column by a + delimiter. Default is "array". + `delimiter` (optional): the delimiter to use when exploding a string column. + This must be provided when `version` is "string" and must be None + when `version` is "array". + `keep`: whether to keep the `data` column in the output alongside + the exploded values. If False, only the exploded values will be + kept. Default is False. + `filtering` (optional): if True, indicates that the explode + operation can result in not every row from the original being + included in the output, i.e. if some of the rows from `data` are + empty arrays. Default is True. + `is_distinct` (optional): if True, indicates that each value within + `data` is unique within that row of the original data. If so, then + the index column is no longer mandatory since the values of the + exploded data can be used to co-identify unique rows. Default is + False. + """ + match version: + case "array": + if delimiter is not None: + raise PyDoughUnqualifiedException( + "Cannot provide a delimiter when version is 'array'" + ) + case "string": + if delimiter is None: + raise PyDoughUnqualifiedException( + "Must provide a delimiter when version is 'string'" + ) + case _: + raise PyDoughUnqualifiedException( + f"Unrecognized version for EXPLODE: {version!r}" + ) + if index_name is None and not is_distinct: + raise PyDoughUnqualifiedException( + "Must provide index_name when is_distinct is False" + ) + return UnqualifiedExplode( + self, + data, + value_name, + index_name, + version, + delimiter, + keep, + filtering, + is_distinct, + ) + class UnqualifiedRoot(UnqualifiedNode): """ @@ -792,10 +867,45 @@ class UnqualifiedGeneratedCollection(UnqualifiedNode): def __init__(self, user_collection: PyDoughUserGeneratedCollection): self._parcel: tuple[PyDoughUserGeneratedCollection] = (user_collection,) - @property - def user_collection(self) -> PyDoughUserGeneratedCollection: - """The wrapped user-generated collection.""" - return self._parcel[0] + +class UnqualifiedExplode(UnqualifiedNode): + """ + Implementation of UnqualifiedNode used to refer to an EXPLODE clause. + """ + + def __init__( + self, + predecessor: UnqualifiedNode, + data: UnqualifiedNode, + value_name: str, + index_name: str | None, + version: str, + delimiter: str | None, + keep: bool, + filtering: bool, + is_distinct: bool, + ): + self._parcel: tuple[ + UnqualifiedNode, + UnqualifiedNode, + str, + str | None, + str, + str | None, + bool, + bool, + bool, + ] = ( + predecessor, + data, + value_name, + index_name, + version, + delimiter, + keep, + filtering, + is_distinct, + ) def display_raw(unqualified: UnqualifiedNode) -> str: @@ -901,6 +1011,19 @@ def display_raw(unqualified: UnqualifiedNode) -> str: result += f"columns=[{', '.join(unqualified._parcel[0].columns)}]," result += f"data={unqualified._parcel[0].to_string()}" return result + ")" + case UnqualifiedExplode(): + result = f"{display_raw(unqualified._parcel[0])}.EXPLODE(" + result += display_raw(unqualified._parcel[1]) + result += f", value_name={unqualified._parcel[2]!r}" + if unqualified._parcel[3] is not None: + result += f", index_name={unqualified._parcel[3]!r}" + result += f", version={unqualified._parcel[4]!r}" + if unqualified._parcel[4] == "string": + result += f", delimiter={unqualified._parcel[5]!r}" + result += f", keep={unqualified._parcel[6]}" + result += f", filtering={unqualified._parcel[7]}" + result += f", is_distinct={unqualified._parcel[8]}" + return result + ")" case _: raise PyDoughUnqualifiedException( f"Unsupported unqualified node: {unqualified.__class__.__name__}" diff --git a/tests/test_pipeline_tpch_custom.py b/tests/test_pipeline_tpch_custom.py index efdbcb9de..404aefa33 100644 --- a/tests/test_pipeline_tpch_custom.py +++ b/tests/test_pipeline_tpch_custom.py @@ -4241,8 +4241,9 @@ "array_data = regions.CALCULATE(nation_names=LISTOF(nations.name))\n" "result = (" " array_data" - " .EXPLODE(nation_names, index_name='nation_idx', value_name='nation_name', keep=True, filtering=False, is_distinct=True)" - " .ORDER_BY(region_name, nation_idx)", + " .EXPLODE(nation_names, index_name='nation_idx', value_name='nation_name', version='array', keep=True, filtering=False, is_distinct=True)" + " .ORDER_BY(region_name, nation_idx)" + ")", "TPCH", lambda: pd.DataFrame( { @@ -4310,10 +4311,49 @@ ), "explode_01", order_sensitive=True, - skipped_dialects={"ANSI", "SQLITE"}, + skipped_dialects={"ANSI", "SQLITE", "MYSQL"}, ), id="explode_01", ), + pytest.param( + PyDoughPandasTest( + "array_data = pydough.dataframe_collection(name='tbl', dataframe=array_df, unique_column_names=['key'])\n" + "result = (" + " array_data" + " .EXPLODE(arr, index_name='arr_idx', value_name='arr_val', version='array', keep=True, filtering=True, is_distinct=True)" + " .ORDER_BY(key, arr_idx)" + ")", + "TPCH", + lambda: pd.DataFrame( + { + "key": ["A", "C", "C", "C", "C", "D", "D"], + "arr": [ + [1], + [2, 3, None, 4], + [2, 3, None, 4], + [2, 3, None, 4], + [2, 3, None, 4], + [5, 6], + [5, 6], + ], + "arr_idx": [1, 1, 2, 3, 4, 1, 2], + "nation_name": [1, 2, 3, None, 4, 5, 6], + } + ), + "explode_02", + order_sensitive=True, + skipped_dialects={"ANSI", "SQLITE"}, + kwargs={ + "array_df": pd.DataFrame( + { + "key": ["A", "B", "C", "D"], + "arr": [[1], [], [2, 3, None, 4], [5, 6]], + } + ) + }, + ), + id="explode_02", + ), pytest.param( PyDoughPandasTest( simple_range_1, diff --git a/tests/test_unqualified_node.py b/tests/test_unqualified_node.py index eda2158c7..378c6194e 100644 --- a/tests/test_unqualified_node.py +++ b/tests/test_unqualified_node.py @@ -326,6 +326,16 @@ def verify_pydough_code_exec_match_unqualified( "customers.CROSS(customers.orders.WHERE((order_priority == '1-URGENT'))).TOP_K(5, by=(key.DESC(na_pos='last')))", id="cross_filter", ), + pytest.param( + "answer = _ROOT.customers.EXPLODE(_ROOT.name, index_name='i', value_name='v', version='string', delimiter=' ')", + "customers.EXPLODE(name, value_name='v', index_name='i', version='string', delimiter=' ', keep=False, filtering=True, is_distinct=False)", + id="explode_01", + ), + pytest.param( + "answer = _ROOT.customers.EXPLODE(_ROOT.name, index_name='i', value_name='v')", + "customers.EXPLODE(name, value_name='v', index_name='i', version='array', keep=False, filtering=True, is_distinct=False)", + id="explode_02", + ), ], ) def test_unqualified_to_string( From 3f6fb6f2a08d4aef83dbe59bf74b7c450eecbe82 Mon Sep 17 00:00:00 2001 From: knassre-bodo Date: Thu, 23 Jul 2026 10:48:27 -0700 Subject: [PATCH 14/44] WIP hybrid/relational translation of EXPLODE operator --- pydough/conversion/hybrid_decorrelater.py | 13 + pydough/conversion/hybrid_operations.py | 42 +++ pydough/conversion/hybrid_translator.py | 54 ++++ pydough/conversion/hybrid_tree.py | 8 + pydough/conversion/relational_converter.py | 19 ++ pydough/qdag/__init__.py | 2 + pydough/qdag/collections/__init__.py | 2 + pydough/qdag/collections/explode.py | 281 ++++++++++++++++++ pydough/qdag/node_builder.py | 48 +++ pydough/relational/__init__.py | 2 + .../relational/relational_nodes/__init__.py | 2 + .../relational/relational_nodes/explode.py | 68 +++++ pydough/unqualified/qualification.py | 63 +++- pydough/unqualified/unqualified_node.py | 57 ++-- pydough/unqualified/unqualified_transform.py | 2 - tests/test_pipeline_tpch_custom.py | 65 +++- .../simple_pydough_functions.py | 118 ++++++++ tests/test_qualification.py | 95 ++++++ tests/test_qualification_errors.py | 110 +++++++ tests/test_unqualified_node.py | 8 +- 20 files changed, 1029 insertions(+), 30 deletions(-) create mode 100644 pydough/qdag/collections/explode.py create mode 100644 pydough/relational/relational_nodes/explode.py diff --git a/pydough/conversion/hybrid_decorrelater.py b/pydough/conversion/hybrid_decorrelater.py index be325a618..4232c64ac 100644 --- a/pydough/conversion/hybrid_decorrelater.py +++ b/pydough/conversion/hybrid_decorrelater.py @@ -27,6 +27,7 @@ from .hybrid_operations import ( HybridCalculate, HybridChildPullUp, + HybridExplode, HybridFilter, HybridNoop, HybridPartition, @@ -307,6 +308,14 @@ def correl_ref_purge( correl_level, new_parent_uni_keys, ) + if isinstance(operation, HybridExplode): + operation.explode_data = self.remove_correl_refs( + operation.explode_data, + old_parent, + child_height, + correl_level, + new_parent_uni_keys, + ) # Repeat the process on the ancestor until either loop guard # condition is no longer True. Only update the child height if we # are still making steps from the original tree, as opposed to from @@ -554,6 +563,10 @@ def find_correlated_children(self, hybrid: HybridTree) -> None: correl_levels = max( correl_levels, operation.condition.count_correlated_levels() ) + if isinstance(operation, HybridExplode): + correl_levels = max( + correl_levels, operation.explode_data.count_correlated_levels() + ) assert correl_levels <= len(self.stack) for i in range(-1, -correl_levels - 1, -1): diff --git a/pydough/conversion/hybrid_operations.py b/pydough/conversion/hybrid_operations.py index 452147758..54895e802 100644 --- a/pydough/conversion/hybrid_operations.py +++ b/pydough/conversion/hybrid_operations.py @@ -9,6 +9,7 @@ "HybridCalculate", "HybridChildPullUp", "HybridCollectionAccess", + "HybridExplode", "HybridFilter", "HybridLimit", "HybridNoop", @@ -31,6 +32,7 @@ from pydough.qdag.collections.user_collection_qdag import ( PyDoughUserGeneratedCollectionQDag, ) +from pydough.types import NumericType from .hybrid_connection import HybridConnection from .hybrid_expressions import ( @@ -518,3 +520,43 @@ def user_collection(self) -> PyDoughUserGeneratedCollectionQDag: def __repr__(self): return self.user_collection.to_string() + + +class HybridExplode(HybridOperation): + """ + Class for HybridOperation corresponding to the EXPLODE operator. + """ + + def __init__( + self, + explode_data: HybridExpr, + value_name: str, + index_name: str | None, + version: str, + delimiter: str | None, + filtering: bool, + is_distinct: bool, + parent_unique: list[HybridExpr], + ): + self.explode_data: HybridExpr = explode_data + self.value_name: str = value_name + self.index_name: str | None = index_name + self.version: str = version + self.delimiter: str | None = delimiter + self.filtering: bool = filtering + self.is_distinct: bool = is_distinct + terms: dict[str, HybridExpr] = {} + unique_exprs: list[HybridExpr] = [] + terms[value_name] = HybridRefExpr(value_name, explode_data.typ) + if index_name is not None: + terms[index_name] = HybridRefExpr(index_name, NumericType()) + if is_distinct: + unique_exprs.append(terms[value_name]) + else: + assert index_name is not None + unique_exprs.append(terms[index_name]) + unique_exprs.extend(parent_unique) + super().__init__(terms, {}, [], unique_exprs) + + def __repr__(self): + return f"EXPLODE[{self.explode_data}, {self.value_name}, {self.index_name}, {self.version}, {self.delimiter}, {self.filtering}, {self.is_distinct}]" diff --git a/pydough/conversion/hybrid_translator.py b/pydough/conversion/hybrid_translator.py index c61a77420..87fb9ca0f 100644 --- a/pydough/conversion/hybrid_translator.py +++ b/pydough/conversion/hybrid_translator.py @@ -26,6 +26,7 @@ ChildReferenceExpression, CollationExpression, ColumnProperty, + Explode, ExpressionFunctionCall, GlobalContext, Literal, @@ -69,6 +70,7 @@ from .hybrid_operations import ( HybridCalculate, HybridCollectionAccess, + HybridExplode, HybridFilter, HybridLimit, HybridNoop, @@ -1335,6 +1337,9 @@ def define_root_link( case HybridRoot(): # A root does not need to be joined to its parent join_keys = [] + case HybridExplode(): + # An explode operator does not need to be joined to its parent + join_keys = [] case HybridUserGeneratedCollection(): # A user-generated collection does not need to be joined to its parent join_keys = [] @@ -1448,6 +1453,26 @@ def make_hybrid_tree( ) hybrid.add_successor(successor_hybrid) return successor_hybrid + case Explode(): + hybrid = self.make_hybrid_tree( + node.ancestor_context, parent, is_aggregate + ) + expr = self.make_hybrid_expr( + hybrid, node.data, child_ref_mapping, False + ) + explode_operator = HybridExplode( + expr.shift_back(1), + node.value_name, + node.index_name, + node.version, + node.delimiter, + node.filtering, + node.is_distinct, + [term.shift_back(1) for term in hybrid.pipeline[-1].unique_exprs], + ) + successor_hybrid = HybridTree(explode_operator, node.ancestral_mapping) + hybrid.add_successor(successor_hybrid) + return successor_hybrid case PartitionChild(): hybrid = self.make_hybrid_tree( node.ancestor_context, parent, is_aggregate @@ -1626,6 +1651,26 @@ def make_hybrid_tree( raise NotImplementedError( f"Unsupported metadata type for subcollection access: {sub_property.__class__.__name__}" ) + case Explode(): + expr = self.make_hybrid_expr( + parent, node.child_access.data, child_ref_mapping, False + ) + explode_operator = HybridExplode( + HybridCorrelExpr(expr), + node.child_access.value_name, + node.child_access.index_name, + node.child_access.version, + node.child_access.delimiter, + node.child_access.filtering, + node.child_access.is_distinct, + [ + HybridCorrelExpr(term) + for term in parent.pipeline[-1].unique_exprs + ], + ) + successor_hybrid = HybridTree( + explode_operator, node.ancestral_mapping + ) case PartitionChild(): source: HybridTree = parent if isinstance(source.pipeline[0], HybridPartitionChild): @@ -1747,8 +1792,14 @@ def convert_qdag_to_hybrid(self, node: PyDoughCollectionQDAG) -> HybridTree: The HybridTree representation of the given QDAG node after transformations. """ + print() + print(node.to_tree_string()) + print() # 1. Run the initial conversion from QDAG to Hybrid hybrid: HybridTree = self.make_hybrid_tree(node, None) + print() + print(hybrid) + print() # 2. Eject any aggregate inputs from the hybrid tree. self.eject_aggregate_inputs(hybrid) # 3. Syncretize any children of the hybrid tree that share a common @@ -1758,6 +1809,9 @@ def convert_qdag_to_hybrid(self, node: PyDoughCollectionQDAG) -> HybridTree: # filters with correlated references into join conditions. self.run_correlation_extraction(hybrid) # 5. Run the de-correlation procedure. + print() + print(hybrid) + print() self.run_hybrid_decorrelation(hybrid) # 6. Run the filter-merging procedure, then re-run ejecting aggregate # inputs to clean up any new aggregates created by filter merging. diff --git a/pydough/conversion/hybrid_tree.py b/pydough/conversion/hybrid_tree.py index c6f9bd220..a24d1866a 100644 --- a/pydough/conversion/hybrid_tree.py +++ b/pydough/conversion/hybrid_tree.py @@ -41,6 +41,7 @@ HybridCalculate, HybridChildPullUp, HybridCollectionAccess, + HybridExplode, HybridFilter, HybridLimit, HybridNoop, @@ -710,6 +711,10 @@ def infer_root_reverse_cardinality(self, context: "HybridTree") -> JoinCardinali ) else: return JoinCardinality.PLURAL_ACCESS + case HybridExplode(): + # Each record of an explode operator points back to one row + # of its parent. + return JoinCardinality.SINGULAR_ACCESS # For partition & partition child, infer from the underlying child. case HybridPartition(): return self.children[0].subtree.infer_root_reverse_cardinality(context) @@ -787,6 +792,9 @@ def always_exists(self) -> bool: ) if not meta.always_matches: return False + case HybridExplode(): + if start_operation.filtering: + return False case HybridPartition(): # For partition nodes, verify the data being partitioned always # exists. diff --git a/pydough/conversion/relational_converter.py b/pydough/conversion/relational_converter.py index 98805e19a..e3a5884e7 100644 --- a/pydough/conversion/relational_converter.py +++ b/pydough/conversion/relational_converter.py @@ -38,6 +38,7 @@ ColumnReference, CorrelatedReference, EmptySingleton, + Explode, ExpressionSortInfo, Filter, GeneratedTable, @@ -82,6 +83,7 @@ HybridCalculate, HybridChildPullUp, HybridCollectionAccess, + HybridExplode, HybridFilter, HybridLimit, HybridNoop, @@ -1402,6 +1404,17 @@ def translate_child_pullup(self, node: HybridChildPullUp) -> TranslationOutput: # expressions mapping return TranslationOutput(child_result.relational_node, new_expressions) + def translate_explode( + self, operation: HybridExplode, context: TranslationOutput + ) -> TranslationOutput: + """ + TODO + """ + raise NotImplementedError() + new_node = Explode() + new_expressions: dict[HybridExpr, ColumnReference] = {} + return TranslationOutput(new_node, new_expressions) + def translate_hybridroot(self, context: TranslationOutput) -> TranslationOutput: """ Converts a HybridRoot node into a relational tree. This method shifts @@ -1582,6 +1595,9 @@ def rel_translation( case HybridLimit(): assert context is not None, "Malformed HybridTree pattern." result = self.translate_limit(operation, context) + case HybridExplode(): + assert context is not None, "Malformed HybridTree pattern." + result = self.translate_explode(operation, context) case HybridChildPullUp(): assert context is None, "Malformed HybridTree pattern." result = self.translate_child_pullup(operation) @@ -1887,6 +1903,9 @@ def convert_ast_to_relational( hybrid_translator: HybridTranslator = HybridTranslator(session) hybrid: HybridTree = hybrid_translator.convert_qdag_to_hybrid(node) + print() + print(hybrid) + # Then, invoke relational conversion procedure. The first element in the # returned list is the final relational tree. output: TranslationOutput = rel_translator.rel_translation( diff --git a/pydough/qdag/__init__.py b/pydough/qdag/__init__.py index 809e7f1e0..0b795691c 100644 --- a/pydough/qdag/__init__.py +++ b/pydough/qdag/__init__.py @@ -16,6 +16,7 @@ "CollationExpression", "CollectionAccess", "ColumnProperty", + "Explode", "ExpressionFunctionCall", "GlobalContext", "Literal", @@ -47,6 +48,7 @@ ChildOperatorChildAccess, ChildReferenceCollection, CollectionAccess, + Explode, GlobalContext, OrderBy, PartitionBy, diff --git a/pydough/qdag/collections/__init__.py b/pydough/qdag/collections/__init__.py index b7574bc6b..487b9195e 100644 --- a/pydough/qdag/collections/__init__.py +++ b/pydough/qdag/collections/__init__.py @@ -11,6 +11,7 @@ "ChildOperatorChildAccess", "ChildReferenceCollection", "CollectionAccess", + "Explode", "GlobalContext", "OrderBy", "PartitionBy", @@ -33,6 +34,7 @@ from .child_reference_collection import ChildReferenceCollection from .collection_access import CollectionAccess from .collection_qdag import PyDoughCollectionQDAG +from .explode import Explode from .global_context import GlobalContext from .order_by import OrderBy from .partition_by import PartitionBy diff --git a/pydough/qdag/collections/explode.py b/pydough/qdag/collections/explode.py new file mode 100644 index 000000000..1c00b5586 --- /dev/null +++ b/pydough/qdag/collections/explode.py @@ -0,0 +1,281 @@ +""" +Base definition of PyDough QDAG collection type for the explode operation. +""" + +__all__ = ["Explode"] + + +import pydough +from pydough.errors import PyDoughQDAGException +from pydough.qdag.abstract_pydough_qdag import PyDoughQDAG +from pydough.qdag.expressions import ( + BackReferenceExpression, + CollationExpression, + PyDoughExpressionQDAG, + Reference, +) +from pydough.types import ArrayType, NumericType, PyDoughType, UnknownType + +from .child_access import ChildAccess +from .collection_qdag import PyDoughCollectionQDAG + + +class Explode(ChildAccess): + """ + The QDAG node implementation class representing an explode operation. + """ + + def __init__( + self, + ancestor: PyDoughCollectionQDAG, + data: PyDoughExpressionQDAG, + name: str, + value_name: str, + index_name: str | None, + version: str, + delimiter: str | None, + filtering: bool, + is_distinct: bool, + ): + super().__init__(ancestor) + if value_name in ancestor.all_terms: + raise PyDoughQDAGException( + f"Cannot use {value_name!r} as the `value_name` for EXPLODE because it is already a term in the ancestor context" + ) + if index_name is not None and index_name in ancestor.all_terms: + raise PyDoughQDAGException( + f"Cannot use {index_name!r} as the `index_name` for EXPLODE because it is already a term in the ancestor context" + ) + self._name: str = name + self._data: PyDoughExpressionQDAG = data + self._value_name: str = value_name + self._index_name: str | None = index_name + self._version: str = version + self._delimiter: str | None = delimiter + self._filtering: bool = filtering + self._is_distinct: bool = is_distinct + self._all_property_names: set[str] = set() + # Build the current node's ancestral mapping by copying the ancestor's + # mapping and incrementing each level by 1 to reflect + # the added depth of this node. + self._ancestral_mapping: dict[str, int] = { + name: level + 1 for name, level in ancestor.ancestral_mapping.items() + } + self._all_property_names.update(self._ancestral_mapping) + self._all_property_names.add(self._value_name) + if self._index_name is not None: + self._all_property_names.add(self._index_name) + + def clone_with_parent(self, new_parent: PyDoughCollectionQDAG) -> "Explode": + return Explode( + new_parent, + self._data, + self._name, + self._value_name, + self._index_name, + self._version, + self._delimiter, + self._filtering, + self._is_distinct, + ) + + @property + def data(self) -> PyDoughExpressionQDAG: + """ + The data that will be exploded by the operation. + """ + return self._data + + @property + def name(self) -> str: + """ + The name of the collection after being exploded. This is the name that + will be used to reference the collection in subsequent operations, such + as window functions. + """ + return self._name + + @property + def key(self) -> str: + return f"{self.ancestor_context.key}.EXPLODE" + + @property + def value_name(self) -> str: + """ + The name of the property that will hold the exploded values. + """ + return self._value_name + + @property + def index_name(self) -> str | None: + """ + The name of the property that will hold the exploded indices, or None + if no index property is being created. + """ + return self._index_name + + @property + def version(self) -> str: + """ + The version of the explode operation, either "array" or "string". + """ + return self._version + + @property + def delimiter(self) -> str | None: + """ + The delimiter to use when exploding a string column, or None if the + version is "array". + """ + return self._delimiter + + @property + def filtering(self) -> bool: + """ + Whether the explode operation can result in not every row from the + original being included in the output, i.e. if some of the rows from + `data` are empty arrays or empty strings. + """ + return self._filtering + + @property + def is_distinct(self) -> bool: + """ + Whether each exploded value will be unique within the output with + regards to the original row from which it was exploded. + """ + return self._is_distinct + + @property + def calc_terms(self) -> set[str]: + if self._index_name is None: + return {self._value_name} + else: + return {self._value_name, self._index_name} + + @property + def all_terms(self) -> set[str]: + return self._all_property_names + + @property + def ancestral_mapping(self) -> dict[str, int]: + return self._ancestral_mapping + + @property + def inherited_downstreamed_terms(self) -> set[str]: + return self.ancestor_context.inherited_downstreamed_terms + + @property + def ordering(self) -> list[CollationExpression] | None: + return None + + @property + def unique_terms(self) -> list[str]: + if self.is_distinct: + return [self.value_name] + else: + assert self.index_name is not None + return [self.index_name] # Note: must add ancestral unique terms + + def is_singular(self, context: PyDoughCollectionQDAG) -> bool: + return False + + def get_expression_position(self, expr_name: str) -> int: + if expr_name == self._value_name: + return 0 + elif expr_name == self._index_name: + return 1 + else: + raise PyDoughQDAGException(f"Unrecognized term of {self!r}: {expr_name!r}") + + def get_term(self, term_name: str) -> PyDoughQDAG: + if term_name not in self.all_terms: + if term_name in self.ancestor_context.all_terms: + result: PyDoughQDAG = self.ancestor_context.get_term(term_name) + if isinstance(result, PyDoughExpressionQDAG): + if isinstance(result, BackReferenceExpression): + return BackReferenceExpression( + self, term_name, result.back_levels + 1 + ) + return BackReferenceExpression(self, term_name, 1) + else: + return result + else: + raise pydough.active_session.error_builder.term_not_found( + collection=self, term_name=term_name + ) + + # Special handling of terms down-streamed from an ancestor CALCULATE + # clause. + if term_name in self.ancestral_mapping: + # Verify that the ancestor name is not also a name in the current + # context. + if term_name in self.calc_terms: + raise pydough.active_session.error_builder.downstream_conflict( + collection=self, term_name=term_name + ) + # Create a back-reference to the ancestor term. + return BackReferenceExpression( + self, term_name, self.ancestral_mapping[term_name] + ) + + if term_name in self.inherited_downstreamed_terms: + context: PyDoughCollectionQDAG = self + while term_name not in context.all_terms: + if context is self: + context = self.ancestor_context + else: + assert context.ancestor_context is not None + context = context.ancestor_context + return Reference( + context, term_name, context.get_expr(term_name).pydough_type + ) + + typ: PyDoughType + if term_name == self._value_name: + if isinstance(self._data.pydough_type, ArrayType): + typ = self._data.pydough_type.elem_type + else: + typ = UnknownType() + else: + assert term_name == self._index_name + typ = NumericType() + + return Reference(self, term_name, typ) + + def to_string(self) -> str: + return f"{self.ancestor_context.to_string()}.{self.standalone_string}" + + @property + def standalone_string(self) -> str: + terms: list[str] = [ + f"EXPLODE[{self._data.to_string()}", + f"name={self._name!r}", + f"value_name={self._value_name!r}", + ] + if self._index_name is not None: + terms.append(f"index_name={self._index_name!r}") + terms.append(f"version={self._version!r}") + if self._version == "string": + terms.append(f"delimiter={self._delimiter!r}") + terms.append(f"filtering={self._filtering}") + terms.append(f"is_distinct={self._is_distinct})") + return ", ".join(terms) + + @property + def tree_item_string(self) -> str: + base_str: str = self.standalone_string + return f"EXPLODE[{base_str[8:-1]}]" + + def equals(self, other: object) -> bool: + return ( + isinstance(other, Explode) + and super().equals(other) + and self.data == other.data + and self.value_name == other.value_name + and self.index_name == other.index_name + and self.version == other.version + and self.delimiter == other.delimiter + and self.filtering == other.filtering + and self.is_distinct == other.is_distinct + ) diff --git a/pydough/qdag/node_builder.py b/pydough/qdag/node_builder.py index 06b7fcb30..812257793 100644 --- a/pydough/qdag/node_builder.py +++ b/pydough/qdag/node_builder.py @@ -28,6 +28,7 @@ Calculate, ChildAccess, ChildReferenceCollection, + Explode, GlobalContext, OrderBy, PartitionBy, @@ -401,6 +402,53 @@ def build_singular( """ return Singular(preceding_context) + def build_explode( + self, + preceding_context: PyDoughCollectionQDAG, + data: PyDoughExpressionQDAG, + name: str, + value_name: str, + index_name: str | None, + version: str, + delimiter: str | None, + filtering: bool, + is_distinct: bool, + ): + """ + Creates an EXPLODE instance. + + Args: + `preceding_context`: the preceding collection. + `data`: the data to be exploded. + `name`: the name of the collection after being exploded. + `value_name`: the name of the value column in the exploded + collection. + `index_name`: the name of the index column in the exploded + collection. + `version`: the version of the explode operation (e.g., "string" or + "array"). + `delimiter`: the delimiter used for string explosion (if + applicable). + `filtering`: whether the explode operation is filtering (i.e., some + rows may be dropped). + `is_distinct`: whether the exploded values are distinct with regards + to the original row. + + Returns: + The newly created PyDough EXPLODE instance. + """ + return Explode( + preceding_context, + data, + name, + value_name, + index_name, + version, + delimiter, + filtering, + is_distinct, + ) + def build_generated_collection( self, preceding_context: PyDoughCollectionQDAG, diff --git a/pydough/relational/__init__.py b/pydough/relational/__init__.py index 161a43c85..d7eed873d 100644 --- a/pydough/relational/__init__.py +++ b/pydough/relational/__init__.py @@ -7,6 +7,7 @@ "ColumnReferenceInputNameModifier", "CorrelatedReference", "EmptySingleton", + "Explode", "ExpressionSortInfo", "Filter", "GeneratedTable", @@ -47,6 +48,7 @@ Aggregate, ColumnPruner, EmptySingleton, + Explode, Filter, GeneratedTable, Join, diff --git a/pydough/relational/relational_nodes/__init__.py b/pydough/relational/relational_nodes/__init__.py index a4e673190..b18079e66 100644 --- a/pydough/relational/relational_nodes/__init__.py +++ b/pydough/relational/relational_nodes/__init__.py @@ -7,6 +7,7 @@ "Aggregate", "ColumnPruner", "EmptySingleton", + "Explode", "Filter", "GeneratedTable", "Join", @@ -27,6 +28,7 @@ from .aggregate import Aggregate from .column_pruner import ColumnPruner from .empty_singleton import EmptySingleton +from .explode import Explode from .filter import Filter from .generated_table import GeneratedTable from .join import Join, JoinCardinality, JoinType diff --git a/pydough/relational/relational_nodes/explode.py b/pydough/relational/relational_nodes/explode.py new file mode 100644 index 000000000..7790384ed --- /dev/null +++ b/pydough/relational/relational_nodes/explode.py @@ -0,0 +1,68 @@ +""" +This file contains the relational implementation for an "explode" operation. +This is our relational representation statements that map to an equivalent of +LATERAL(FLATTEN(...)) +""" + +from typing import TYPE_CHECKING + +from pydough.relational.relational_expressions import RelationalExpression + +from .abstract_node import RelationalNode +from .single_relational import SingleRelational + +if TYPE_CHECKING: + from .relational_shuttle import RelationalShuttle + from .relational_visitor import RelationalVisitor + + +class Explode(SingleRelational): + """ + The Explode node in the relational tree. + """ + + def __init__( + self, + input: RelationalNode, + explode_data: RelationalExpression, + value_name: str, + index_name: str | None, + version: str, + delimiter: str | None, + filtering: bool, + is_distinct: bool, + columns: dict[str, RelationalExpression], + ) -> None: + super().__init__(input, columns) + self._explode_data: RelationalExpression = explode_data + + @property + def explode_data(self) -> RelationalExpression: + """ + The data being exploded. + """ + return self._explode_data + + def node_equals(self, other: RelationalNode) -> bool: + return ( + isinstance(other, Explode) + and self.explode_data == other.explode_data + and super().node_equals(other) + ) + + def to_string(self, compact: bool = False) -> str: + return "Explode(...)" + + def accept(self, visitor: "RelationalVisitor") -> None: + raise NotImplementedError() + visitor.visit_filter(self) + + def accept_shuttle(self, shuttle: "RelationalShuttle") -> RelationalNode: + raise NotImplementedError() + + def node_copy( + self, + columns: dict[str, RelationalExpression], + inputs: list[RelationalNode], + ) -> RelationalNode: + raise NotImplementedError() diff --git a/pydough/unqualified/qualification.py b/pydough/unqualified/qualification.py index cde40330d..8efa34c2f 100644 --- a/pydough/unqualified/qualification.py +++ b/pydough/unqualified/qualification.py @@ -10,7 +10,7 @@ import pydough import pydough.pydough_operators as pydop from pydough.configs import PyDoughSession -from pydough.errors import PyDoughUnqualifiedException +from pydough.errors import PyDoughQDAGException, PyDoughUnqualifiedException from pydough.metadata import GeneralJoinMetadata, GraphMetadata from pydough.pydough_operators.expression_operators import ( ExpressionFunctionOperator, @@ -43,6 +43,7 @@ UnqualifiedCalculate, UnqualifiedCollation, UnqualifiedCross, + UnqualifiedExplode, UnqualifiedGeneratedCollection, UnqualifiedLiteral, UnqualifiedNode, @@ -546,6 +547,7 @@ def qualify_join_condition( | UnqualifiedWhere() | UnqualifiedOrderBy() | UnqualifiedTopK() + | UnqualifiedExplode() ): raise PyDoughUnqualifiedException( "Collection accesses are currently unsupported in PyDough general join conditions" @@ -907,6 +909,7 @@ def split_partition_ancestry( | UnqualifiedWhere() | UnqualifiedTopK() | UnqualifiedOrderBy() + | UnqualifiedExplode() | UnqualifiedSingular() | UnqualifiedPartition() | UnqualifiedBest() @@ -985,6 +988,8 @@ def split_partition_ancestry( build_node[0] = UnqualifiedBest(build_node[0], *node._parcel[1:]) case UnqualifiedCross(): build_node[0] = UnqualifiedCross(build_node[0], *node._parcel[1:]) + case UnqualifiedExplode(): + build_node[0] = UnqualifiedExplode(build_node[0], *node._parcel[1:]) case _: # Any other unqualified node would mean something is malformed. raise PyDoughUnqualifiedException( @@ -1272,6 +1277,60 @@ def qualify_cross( return qualified_child + def qualify_explode( + self, + unqualified: UnqualifiedExplode, + context: PyDoughCollectionQDAG, + is_child: bool, + ) -> PyDoughCollectionQDAG: + """ + Transforms an `UnqualifiedExplode` into a PyDoughCollectionQDAG node. + + Args: + `unqualified`: the UnqualifiedExplode instance to be transformed. + `context`: the collection QDAG whose context the collection is being + evaluated within. + `is_child`: whether the collection is being qualified as a child + of a child operator context, such as CALCULATE or PARTITION. + """ + unqualified_parent: UnqualifiedNode = unqualified._parcel[0] + data_raw: UnqualifiedNode = unqualified._parcel[1] + name: str = unqualified._parcel[2] + value_name: str = unqualified._parcel[3] + index_name: str | None = unqualified._parcel[4] + version: str = unqualified._parcel[5] + delimiter: str | None = unqualified._parcel[6] + filtering: bool = unqualified._parcel[7] + is_distinct: bool = unqualified._parcel[8] + + qualified_parent: PyDoughCollectionQDAG = self.qualify_collection( + unqualified_parent, context, is_child + ) + + # Qualify the data being explored with regards to the table being + # exploded storing any children built along the way. + children: list[PyDoughCollectionQDAG] = [] + qualified_data = self.qualify_expression(data_raw, qualified_parent, children) + if len(children) > 0: + raise PyDoughQDAGException( + f"Invalid data argument to explode: {data_raw!r} (explode does not currently support exploding data from a child collection; make sure to store the data to be exploded in a column of the parent collection before calling explode)" + ) + # Use the qualified children & terms to create a new EXPLODE node. + answer: PyDoughCollectionQDAG = self.builder.build_explode( + qualified_parent, + qualified_data, + name, + value_name, + index_name, + version, + delimiter, + filtering, + is_distinct, + ) + if isinstance(unqualified_parent, UnqualifiedRoot) and is_child: + answer = ChildOperatorChildAccess(answer) + return answer + def qualify_generated_collection( self, unqualified: UnqualifiedGeneratedCollection, @@ -1369,6 +1428,8 @@ def qualify_node( answer = self.qualify_best(unqualified, context, is_child) case UnqualifiedCross(): answer = self.qualify_cross(unqualified, context, is_child) + case UnqualifiedExplode(): + answer = self.qualify_explode(unqualified, context, is_child) case UnqualifiedGeneratedCollection(): answer = self.qualify_generated_collection( unqualified, context, is_child diff --git a/pydough/unqualified/unqualified_node.py b/pydough/unqualified/unqualified_node.py index 39fdb39d5..4225aade4 100644 --- a/pydough/unqualified/unqualified_node.py +++ b/pydough/unqualified/unqualified_node.py @@ -438,11 +438,11 @@ def BEST( def EXPLODE( self, data: "UnqualifiedNode", + name: str, value_name: str, index_name: str | None = None, version: str = "array", delimiter: str | None = None, - keep: bool = False, filtering: bool = True, is_distinct: bool = False, ): @@ -453,6 +453,9 @@ def EXPLODE( Args: `data`: the array or string column to explode. + `name`: the name of the collection after being exploded. + This is the name that will be used to reference the collection in + subsequent operations, such as window functions. `value_name`: the name of the column representing the exploded values. `index_name` (optional): the name of the column representing the @@ -466,9 +469,6 @@ def EXPLODE( `delimiter` (optional): the delimiter to use when exploding a string column. This must be provided when `version` is "string" and must be None when `version` is "array". - `keep`: whether to keep the `data` column in the output alongside - the exploded values. If False, only the exploded values will be - kept. Default is False. `filtering` (optional): if True, indicates that the explode operation can result in not every row from the original being included in the output, i.e. if some of the rows from `data` are @@ -483,29 +483,46 @@ def EXPLODE( case "array": if delimiter is not None: raise PyDoughUnqualifiedException( - "Cannot provide a delimiter when version is 'array'" + "Cannot provide a `delimiter` to EXPLODE when `version` is 'array'" ) case "string": - if delimiter is None: + if ( + delimiter is None + or not isinstance(delimiter, str) + or len(delimiter) == 0 + ): raise PyDoughUnqualifiedException( - "Must provide a delimiter when version is 'string'" + "Must provide a non-empty string `delimiter` to EXPLODE when `version` is 'string'" ) case _: raise PyDoughUnqualifiedException( - f"Unrecognized version for EXPLODE: {version!r}" + f"Unrecognized `version` for EXPLODE: {version!r} (must be either 'array' or 'string')" ) + assert isinstance(name, str), f"Invalid `name` argument for EXPLODE: {name!r}" + assert isinstance(value_name, str), ( + f"Invalid `value_name` argument for EXPLODE: {value_name!r}" + ) + assert index_name is None or isinstance(index_name, str), ( + f"Invalid `index_name` argument for EXPLODE: {index_name!r}" + ) + assert isinstance(filtering, bool), ( + f"Invalid `filtering` argument for EXPLODE: {filtering!r}" + ) + assert isinstance(is_distinct, bool), ( + f"Invalid `is_distinct` argument for EXPLODE: {is_distinct!r}" + ) if index_name is None and not is_distinct: raise PyDoughUnqualifiedException( - "Must provide index_name when is_distinct is False" + "Must provide `index_name` to EXPLODE when `is_distinct` is False" ) return UnqualifiedExplode( self, data, + name, value_name, index_name, version, delimiter, - keep, filtering, is_distinct, ) @@ -877,11 +894,11 @@ def __init__( self, predecessor: UnqualifiedNode, data: UnqualifiedNode, + name: str, value_name: str, index_name: str | None, version: str, delimiter: str | None, - keep: bool, filtering: bool, is_distinct: bool, ): @@ -889,20 +906,20 @@ def __init__( UnqualifiedNode, UnqualifiedNode, str, + str, str | None, str, str | None, bool, bool, - bool, ] = ( predecessor, data, + name, value_name, index_name, version, delimiter, - keep, filtering, is_distinct, ) @@ -1014,13 +1031,13 @@ def display_raw(unqualified: UnqualifiedNode) -> str: case UnqualifiedExplode(): result = f"{display_raw(unqualified._parcel[0])}.EXPLODE(" result += display_raw(unqualified._parcel[1]) - result += f", value_name={unqualified._parcel[2]!r}" - if unqualified._parcel[3] is not None: - result += f", index_name={unqualified._parcel[3]!r}" - result += f", version={unqualified._parcel[4]!r}" - if unqualified._parcel[4] == "string": - result += f", delimiter={unqualified._parcel[5]!r}" - result += f", keep={unqualified._parcel[6]}" + result += f", name={unqualified._parcel[2]!r}" + result += f", value_name={unqualified._parcel[3]!r}" + if unqualified._parcel[4] is not None: + result += f", index_name={unqualified._parcel[4]!r}" + result += f", version={unqualified._parcel[5]!r}" + if unqualified._parcel[5] == "string": + result += f", delimiter={unqualified._parcel[6]!r}" result += f", filtering={unqualified._parcel[7]}" result += f", is_distinct={unqualified._parcel[8]}" return result + ")" diff --git a/pydough/unqualified/unqualified_transform.py b/pydough/unqualified/unqualified_transform.py index 76a001cbd..bdceec44f 100644 --- a/pydough/unqualified/unqualified_transform.py +++ b/pydough/unqualified/unqualified_transform.py @@ -519,8 +519,6 @@ def from_string( else: # Just set the metadata on the existing session pydough.active_session.metadata = metadata - print(transformed_code) - print(execution_context) exec(compile_ast, {}, execution_context) finally: if session is not None: diff --git a/tests/test_pipeline_tpch_custom.py b/tests/test_pipeline_tpch_custom.py index b66d66c08..befe9d860 100644 --- a/tests/test_pipeline_tpch_custom.py +++ b/tests/test_pipeline_tpch_custom.py @@ -4239,10 +4239,10 @@ ), pytest.param( PyDoughPandasTest( - "array_data = regions.CALCULATE(nation_names=LISTOF(nations.name))\n" + "array_data = regions.CALCULATE(region_name=name, nation_names=LISTOF(nations.name))\n" "result = (" " array_data" - " .EXPLODE(nation_names, index_name='nation_idx', value_name='nation_name', version='array', keep=True, filtering=False, is_distinct=True)" + " .EXPLODE(nation_names, 'exploded_nations', index_name='nation_idx', value_name='nation_name', version='array', filtering=False, is_distinct=True)" " .ORDER_BY(region_name, nation_idx)" ")", "TPCH", @@ -4321,7 +4321,7 @@ "array_data = pydough.dataframe_collection(name='tbl', dataframe=array_df, unique_column_names=['key'])\n" "result = (" " array_data" - " .EXPLODE(arr, index_name='arr_idx', value_name='arr_val', version='array', keep=True, filtering=True, is_distinct=True)" + " .EXPLODE(arr, 'exploded_array', index_name='arr_idx', value_name='arr_val', version='array'filtering=True, is_distinct=True)" " .ORDER_BY(key, arr_idx)" ")", "TPCH", @@ -4355,6 +4355,65 @@ ), id="explode_02", ), + pytest.param( + PyDoughPandasTest( + "result = (" + " customers" + " .TOP_K(5, by=key.ASC())" + " .EXPLODE(name, 'exploded_customers', index_name='idx', value_name='val', version='string', delimiter='#')" + " .ORDER_BY(name, idx)" + ")", + "TPCH", + lambda: pd.DataFrame( + { + "idx": [1, 2] * 5, + "val": [ + "Customer", + "000000001", + "Customer", + "000000002", + "Customer", + "000000003", + "Customer", + "000000004", + "Customer", + "000000005", + ], + } + ), + "explode_03", + order_sensitive=True, + skipped_dialects={"ANSI", "SQLITE"}, + ), + id="explode_03", + ), + pytest.param( + PyDoughPandasTest( + "exploded_data = EXPLODE(name, 'exploded_names', index_name='idx', value_name='val', version='string', delimiter='I')\n" + "result = (" + " regions" + " .CALCULATE(region_name=name, n_chunks=COUNT(exploded_data))" + " .ORDER_BY(region_name)" + ")", + "TPCH", + lambda: pd.DataFrame( + { + "region_name": [ + "AFRICA", + "AMERICA", + "ASIA", + "EUROPE", + "MIDDLE EAST", + ], + "n_chunks": [2, 2, 2, 1, 1], + } + ), + "explode_04", + order_sensitive=True, + skipped_dialects={"ANSI", "SQLITE"}, + ), + id="explode_04", + ), pytest.param( PyDoughPandasTest( simple_range_1, diff --git a/tests/test_pydough_functions/simple_pydough_functions.py b/tests/test_pydough_functions/simple_pydough_functions.py index f0599f819..477d767c1 100644 --- a/tests/test_pydough_functions/simple_pydough_functions.py +++ b/tests/test_pydough_functions/simple_pydough_functions.py @@ -59,6 +59,124 @@ def dumb_aggregation(): ) +def good_explode_01(): + return nations.EXPLODE( + name, + "exploded_nation", + value_name="v", + index_name="i", + version="array", + filtering=True, + is_distinct=False, + ) + + +def good_explode_02(): + return nations.EXPLODE( + name, + "exploded_nation", + value_name="v", + index_name="i", + version="array", + filtering=True, + is_distinct=True, + ) + + +def good_explode_03(): + return nations.EXPLODE( + name, + "exploded_nation", + value_name="v", + index_name="i", + version="array", + filtering=False, + is_distinct=False, + ) + + +def good_explode_04(): + return nations.EXPLODE( + name, + "exploded_nation", + value_name="v", + index_name="i", + version="array", + filtering=True, + is_distinct=False, + ) + + +def good_explode_05(): + return nations.EXPLODE( + name, + "exploded_nation", + value_name="v", + index_name="i", + version="string", + delimiter=" ", + filtering=True, + is_distinct=False, + ) + + +def good_explode_06(): + return nations.EXPLODE( + name, + "exploded_nation", + value_name="v", + index_name="i", + version="string", + delimiter=" ", + filtering=True, + is_distinct=True, + ) + + +def good_explode_07(): + return nations.EXPLODE( + name, + "exploded_nation", + value_name="v", + index_name="i", + version="string", + delimiter=" ", + filtering=False, + is_distinct=False, + ) + + +def good_explode_08(): + return nations.EXPLODE( + name, + "exploded_nation", + value_name="v", + index_name="i", + version="string", + delimiter=" ", + filtering=True, + is_distinct=False, + ) + + +def good_explode_09(): + return ( + nations.CALCULATE(region_name=region.name) + .EXPLODE( + name, + "exploded_nation", + value_name="v", + index_name="i", + version="string", + delimiter=" ", + filtering=True, + is_distinct=False, + ) + .WHERE(v[:1] != region_name[:1]) + .customers + ) + + def simple_collation(): return ( suppliers.CALCULATE( diff --git a/tests/test_qualification.py b/tests/test_qualification.py index 297fc3448..5825bc0d1 100644 --- a/tests/test_qualification.py +++ b/tests/test_qualification.py @@ -18,6 +18,15 @@ absurd_partition_window_per, absurd_window_per, customer_most_recent_orders, + good_explode_01, + good_explode_02, + good_explode_03, + good_explode_04, + good_explode_05, + good_explode_06, + good_explode_07, + good_explode_08, + good_explode_09, n_orders_first_day, orders_versus_first_orders, partition_as_child, @@ -959,6 +968,92 @@ """, id="simple_range_2", ), + pytest.param( + good_explode_01, + """ +──┬─ TPCH + └─┬─ TableCollection[nations] + └─── EXPLODE[name, name='exploded_nation', value_name='v', index_name='i', version='array', filtering=True, is_distinct=False] + """, + id="good_explode_01", + ), + pytest.param( + good_explode_02, + """ +──┬─ TPCH + └─┬─ TableCollection[nations] + └─── EXPLODE[name, name='exploded_nation', value_name='v', index_name='i', version='array', filtering=True, is_distinct=True] + """, + id="good_explode_02", + ), + pytest.param( + good_explode_03, + """ +──┬─ TPCH + └─┬─ TableCollection[nations] + └─── EXPLODE[name, name='exploded_nation', value_name='v', index_name='i', version='array', filtering=False, is_distinct=False] + """, + id="good_explode_03", + ), + pytest.param( + good_explode_04, + """ +──┬─ TPCH + └─┬─ TableCollection[nations] + └─── EXPLODE[name, name='exploded_nation', value_name='v', index_name='i', version='array', filtering=True, is_distinct=False] + """, + id="good_explode_04", + ), + pytest.param( + good_explode_05, + """ +──┬─ TPCH + └─┬─ TableCollection[nations] + └─── EXPLODE[name, name='exploded_nation', value_name='v', index_name='i', version='string', delimiter=' ', filtering=True, is_distinct=False] + """, + id="good_explode_05", + ), + pytest.param( + good_explode_06, + """ +──┬─ TPCH + └─┬─ TableCollection[nations] + └─── EXPLODE[name, name='exploded_nation', value_name='v', index_name='i', version='string', delimiter=' ', filtering=True, is_distinct=True] + """, + id="good_explode_06", + ), + pytest.param( + good_explode_07, + """ +──┬─ TPCH + └─┬─ TableCollection[nations] + └─── EXPLODE[name, name='exploded_nation', value_name='v', index_name='i', version='string', delimiter=' ', filtering=False, is_distinct=False] + """, + id="good_explode_07", + ), + pytest.param( + good_explode_08, + """ +──┬─ TPCH + └─┬─ TableCollection[nations] + └─── EXPLODE[name, name='exploded_nation', value_name='v', index_name='i', version='string', delimiter=' ', filtering=True, is_distinct=False] + """, + id="good_explode_08", + ), + pytest.param( + good_explode_09, + """ +──┬─ TPCH + ├─── TableCollection[nations] + └─┬─ Calculate[region_name=$1.name] + ├─┬─ AccessChild + │ └─── SubCollection[region] + ├─── EXPLODE[name, name='exploded_nation', value_name='v', index_name='i', version='string', delimiter=' ', filtering=True, is_distinct=False] + └─┬─ Where[SLICE(v, None, 1, None) != SLICE(region_name, None, 1, None)] + └─── SubCollection[customers] + """, + id="good_explode_09", + ), ], ) def test_qualify_node_to_ast_string( diff --git a/tests/test_qualification_errors.py b/tests/test_qualification_errors.py index 0be9a3aaf..e99520aec 100644 --- a/tests/test_qualification_errors.py +++ b/tests/test_qualification_errors.py @@ -200,6 +200,116 @@ "Unrecognized term of TPCH: 'TPCH'. Did you mean: lines, parts, orders, nations, regions?", id="double_graph", ), + pytest.param( + "result = nations.EXPLODE(name, 'names', value_name='v')", + "Must provide `index_name` to EXPLODE when `is_distinct` is False", + id="bad_explode_01", + ), + pytest.param( + "result = nations.EXPLODE()", + "missing 3 required positional arguments: 'data', 'name', and 'value_name'", + id="bad_explode_02", + ), + pytest.param( + "result = nations.EXPLODE(name, 'names')", + "missing 1 required positional argument: 'value_name'", + id="bad_explode_03", + ), + pytest.param( + "result = nations.EXPLODE(name, value_name='v')", + "missing 1 required positional argument: 'name'", + id="bad_explode_04", + ), + pytest.param( + "result = nations.EXPLODE(name, 'names', value_name='v', index_name='i', version='foo')", + "Unrecognized `version` for EXPLODE: 'foo' (must be either 'array' or 'string')", + id="bad_explode_05", + ), + pytest.param( + "result = nations.EXPLODE(name, 'names', value_name='v', index_name='i', version='string')", + "Must provide a non-empty string `delimiter` to EXPLODE when `version` is 'string'", + id="bad_explode_06", + ), + pytest.param( + "result = nations.EXPLODE(name, 'names', value_name='v', index_name='i', version='string', delimiter='')", + "Must provide a non-empty string `delimiter` to EXPLODE when `version` is 'string'", + id="bad_explode_07", + ), + pytest.param( + "result = nations.EXPLODE(name, 'names', value_name='v', index_name='i', version='string', delimiter=42)", + "Must provide a non-empty string `delimiter` to EXPLODE when `version` is 'string'", + id="bad_explode_08", + ), + pytest.param( + "result = nations.EXPLODE(name, 'names', value_name='v', index_name='i', delimiter='x')", + "Cannot provide a `delimiter` to EXPLODE when `version` is 'array'", + id="bad_explode_09", + ), + pytest.param( + "result = nations.EXPLODE(name, 'names', value_name='v', index_name='i', version='array', delimiter='x')", + "Cannot provide a `delimiter` to EXPLODE when `version` is 'array'", + id="bad_explode_10", + ), + pytest.param( + "result = nations.EXPLODE(nation_names, 'names', value_name='v', index_name='i')", + "Unrecognized term of TPCH.nations: 'nation_names'. Did you mean: name, region, region_key?", + id="bad_explode_11", + ), + pytest.param( + "result = nations.EXPLODE(name, 'names', value_name='v', index_name='i', version='string', delimiter=0)", + "Must provide a non-empty string `delimiter` to EXPLODE when `version` is 'string'", + id="bad_explode_12", + ), + pytest.param( + "result = nations.EXPLODE(name, 0, value_name='v', index_name='i', version='string', delimiter=' ')", + "Invalid `name` argument for EXPLODE: 0", + id="bad_explode_13", + ), + pytest.param( + "result = nations.EXPLODE(name, 'names', value_name=0, index_name='i', version='string', delimiter=' ')", + "Invalid `value_name` argument for EXPLODE: 0", + id="bad_explode_14", + ), + pytest.param( + "result = nations.EXPLODE(name, 'names', value_name='v', index_name=0, version='string', delimiter=' ')", + "Invalid `index_name` argument for EXPLODE: 0", + id="bad_explode_15", + ), + pytest.param( + "result = nations.EXPLODE(name, 'names', value_name='v', index_name='i', version=0, delimiter=' ')", + "Unrecognized `version` for EXPLODE: 0 (must be either 'array' or 'string')", + id="bad_explode_16", + ), + pytest.param( + "result = nations.EXPLODE(name, 'names', value_name='v', index_name='i', version='string', delimiter=' ', filtering='yay')", + "Invalid `filtering` argument for EXPLODE: 'yay'", + id="bad_explode_17", + ), + pytest.param( + "result = nations.EXPLODE(name, 'names', value_name='v', index_name='i', version='string', delimiter=' ', is_distinct='nope')", + "Invalid `is_distinct` argument for EXPLODE: 'nope'", + id="bad_explode_18", + ), + pytest.param( + "result = nations.EXPLODE(name, 'names', value_name='v', index_name='i', version='string', delimiter=' ', extra_kwarg='fizzbuzz')", + "UnqualifiedNode.EXPLODE() got an unexpected keyword argument 'extra_kwarg'", + id="bad_explode_19", + ), + pytest.param( + "result = nations.EXPLODE(customers.name, 'names', value_name='v', index_name='i', version='string', delimiter='#')", + "Invalid data argument to explode: customers.name (explode does not currently support exploding data from a child collection; make sure to store the data to be exploded in a column of the parent collection before calling explode)", + id="bad_explode_20", + ), + pytest.param( + "result = nations.EXPLODE(name, 'names', value_name='name', index_name='i', version='string', delimiter='#')", + "Cannot use 'name' as the `value_name` for EXPLODE because it is already a term in the ancestor context", + id="bad_explode_21", + ), + pytest.param( + "result = nations.EXPLODE(name, 'names', value_name='v', index_name='key', version='string', delimiter='#')", + "Cannot use 'key' as the `index_name` for EXPLODE because it is already a term in the ancestor context", + id="bad_explode_22", + ), ], ) def test_qualify_error( diff --git a/tests/test_unqualified_node.py b/tests/test_unqualified_node.py index 378c6194e..8924d7bba 100644 --- a/tests/test_unqualified_node.py +++ b/tests/test_unqualified_node.py @@ -327,13 +327,13 @@ def verify_pydough_code_exec_match_unqualified( id="cross_filter", ), pytest.param( - "answer = _ROOT.customers.EXPLODE(_ROOT.name, index_name='i', value_name='v', version='string', delimiter=' ')", - "customers.EXPLODE(name, value_name='v', index_name='i', version='string', delimiter=' ', keep=False, filtering=True, is_distinct=False)", + "answer = _ROOT.customers.EXPLODE(_ROOT.name, 'exploded_customer', index_name='i', value_name='v', version='string', delimiter=' ')", + "customers.EXPLODE(name, name='exploded_customer', value_name='v', index_name='i', version='string', delimiter=' ', filtering=True, is_distinct=False)", id="explode_01", ), pytest.param( - "answer = _ROOT.customers.EXPLODE(_ROOT.name, index_name='i', value_name='v')", - "customers.EXPLODE(name, value_name='v', index_name='i', version='array', keep=False, filtering=True, is_distinct=False)", + "answer = _ROOT.customers.EXPLODE(_ROOT.name, 'exploded_customer', index_name='i', value_name='v')", + "customers.EXPLODE(name, name='exploded_customer', value_name='v', index_name='i', version='array', filtering=True, is_distinct=False)", id="explode_02", ), ], From 2bdf0312ec8ee8025ca7b02f4ed9d818e2c1f161 Mon Sep 17 00:00:00 2001 From: knassre-bodo Date: Fri, 24 Jul 2026 10:30:48 -0700 Subject: [PATCH 15/44] Switched to ExplodeSpec handling of misc args --- pydough/conversion/hybrid_operations.py | 35 +++---- pydough/conversion/hybrid_translator.py | 14 +-- pydough/conversion/hybrid_tree.py | 2 +- pydough/qdag/collections/explode.py | 134 +++++++----------------- pydough/qdag/node_builder.py | 28 +---- pydough/unqualified/qualification.py | 15 +-- pydough/unqualified/unqualified_node.py | 40 ++----- pydough/utilities/__init__.py | 8 ++ pydough/utilities/explode_spec.py | 45 ++++++++ 9 files changed, 123 insertions(+), 198 deletions(-) create mode 100644 pydough/utilities/__init__.py create mode 100644 pydough/utilities/explode_spec.py diff --git a/pydough/conversion/hybrid_operations.py b/pydough/conversion/hybrid_operations.py index 54895e802..b483e885b 100644 --- a/pydough/conversion/hybrid_operations.py +++ b/pydough/conversion/hybrid_operations.py @@ -33,6 +33,7 @@ PyDoughUserGeneratedCollectionQDag, ) from pydough.types import NumericType +from pydough.utilities import ExplodeSpec from .hybrid_connection import HybridConnection from .hybrid_expressions import ( @@ -530,33 +531,27 @@ class HybridExplode(HybridOperation): def __init__( self, explode_data: HybridExpr, - value_name: str, - index_name: str | None, - version: str, - delimiter: str | None, - filtering: bool, - is_distinct: bool, + explode_spec: ExplodeSpec, parent_unique: list[HybridExpr], ): self.explode_data: HybridExpr = explode_data - self.value_name: str = value_name - self.index_name: str | None = index_name - self.version: str = version - self.delimiter: str | None = delimiter - self.filtering: bool = filtering - self.is_distinct: bool = is_distinct + self.explode_spec: ExplodeSpec = explode_spec terms: dict[str, HybridExpr] = {} unique_exprs: list[HybridExpr] = [] - terms[value_name] = HybridRefExpr(value_name, explode_data.typ) - if index_name is not None: - terms[index_name] = HybridRefExpr(index_name, NumericType()) - if is_distinct: - unique_exprs.append(terms[value_name]) + terms[explode_spec.value_name] = HybridRefExpr( + explode_spec.value_name, explode_data.typ + ) + if explode_spec.index_name is not None: + terms[explode_spec.index_name] = HybridRefExpr( + explode_spec.index_name, NumericType() + ) + if explode_spec.is_distinct: + unique_exprs.append(terms[explode_spec.value_name]) else: - assert index_name is not None - unique_exprs.append(terms[index_name]) + assert explode_spec.index_name is not None + unique_exprs.append(terms[explode_spec.index_name]) unique_exprs.extend(parent_unique) super().__init__(terms, {}, [], unique_exprs) def __repr__(self): - return f"EXPLODE[{self.explode_data}, {self.value_name}, {self.index_name}, {self.version}, {self.delimiter}, {self.filtering}, {self.is_distinct}]" + return f"EXPLODE[{self.explode_data}, {self.explode_spec.arg_list_string}]" diff --git a/pydough/conversion/hybrid_translator.py b/pydough/conversion/hybrid_translator.py index 87fb9ca0f..b7c543e6a 100644 --- a/pydough/conversion/hybrid_translator.py +++ b/pydough/conversion/hybrid_translator.py @@ -1462,12 +1462,7 @@ def make_hybrid_tree( ) explode_operator = HybridExplode( expr.shift_back(1), - node.value_name, - node.index_name, - node.version, - node.delimiter, - node.filtering, - node.is_distinct, + node.explode_spec, [term.shift_back(1) for term in hybrid.pipeline[-1].unique_exprs], ) successor_hybrid = HybridTree(explode_operator, node.ancestral_mapping) @@ -1657,12 +1652,7 @@ def make_hybrid_tree( ) explode_operator = HybridExplode( HybridCorrelExpr(expr), - node.child_access.value_name, - node.child_access.index_name, - node.child_access.version, - node.child_access.delimiter, - node.child_access.filtering, - node.child_access.is_distinct, + node.child_access.explode_spec, [ HybridCorrelExpr(term) for term in parent.pipeline[-1].unique_exprs diff --git a/pydough/conversion/hybrid_tree.py b/pydough/conversion/hybrid_tree.py index a24d1866a..679ff9e88 100644 --- a/pydough/conversion/hybrid_tree.py +++ b/pydough/conversion/hybrid_tree.py @@ -793,7 +793,7 @@ def always_exists(self) -> bool: if not meta.always_matches: return False case HybridExplode(): - if start_operation.filtering: + if start_operation.explode_spec.filtering: return False case HybridPartition(): # For partition nodes, verify the data being partitioned always diff --git a/pydough/qdag/collections/explode.py b/pydough/qdag/collections/explode.py index 1c00b5586..77f522d3c 100644 --- a/pydough/qdag/collections/explode.py +++ b/pydough/qdag/collections/explode.py @@ -15,6 +15,7 @@ Reference, ) from pydough.types import ArrayType, NumericType, PyDoughType, UnknownType +from pydough.utilities import ExplodeSpec from .child_access import ChildAccess from .collection_qdag import PyDoughCollectionQDAG @@ -30,30 +31,23 @@ def __init__( ancestor: PyDoughCollectionQDAG, data: PyDoughExpressionQDAG, name: str, - value_name: str, - index_name: str | None, - version: str, - delimiter: str | None, - filtering: bool, - is_distinct: bool, + explode_spec: ExplodeSpec, ): super().__init__(ancestor) - if value_name in ancestor.all_terms: + if explode_spec.value_name in ancestor.all_terms: raise PyDoughQDAGException( - f"Cannot use {value_name!r} as the `value_name` for EXPLODE because it is already a term in the ancestor context" + f"Cannot use {explode_spec.value_name!r} as the `value_name` for EXPLODE because it is already a term in the ancestor context" ) - if index_name is not None and index_name in ancestor.all_terms: + if ( + explode_spec.index_name is not None + and explode_spec.index_name in ancestor.all_terms + ): raise PyDoughQDAGException( - f"Cannot use {index_name!r} as the `index_name` for EXPLODE because it is already a term in the ancestor context" + f"Cannot use {explode_spec.index_name!r} as the `index_name` for EXPLODE because it is already a term in the ancestor context" ) self._name: str = name self._data: PyDoughExpressionQDAG = data - self._value_name: str = value_name - self._index_name: str | None = index_name - self._version: str = version - self._delimiter: str | None = delimiter - self._filtering: bool = filtering - self._is_distinct: bool = is_distinct + self._explode_spec: ExplodeSpec = explode_spec self._all_property_names: set[str] = set() # Build the current node's ancestral mapping by copying the ancestor's # mapping and incrementing each level by 1 to reflect @@ -62,21 +56,16 @@ def __init__( name: level + 1 for name, level in ancestor.ancestral_mapping.items() } self._all_property_names.update(self._ancestral_mapping) - self._all_property_names.add(self._value_name) - if self._index_name is not None: - self._all_property_names.add(self._index_name) + self._all_property_names.add(explode_spec.value_name) + if explode_spec.index_name is not None: + self._all_property_names.add(explode_spec.index_name) def clone_with_parent(self, new_parent: PyDoughCollectionQDAG) -> "Explode": return Explode( new_parent, - self._data, - self._name, - self._value_name, - self._index_name, - self._version, - self._delimiter, - self._filtering, - self._is_distinct, + self.data, + self.name, + self.explode_spec, ) @property @@ -100,58 +89,19 @@ def key(self) -> str: return f"{self.ancestor_context.key}.EXPLODE" @property - def value_name(self) -> str: + def explode_spec(self) -> ExplodeSpec: """ - The name of the property that will hold the exploded values. + The dataclass payload containing the specifications for the explode + operation. """ - return self._value_name - - @property - def index_name(self) -> str | None: - """ - The name of the property that will hold the exploded indices, or None - if no index property is being created. - """ - return self._index_name - - @property - def version(self) -> str: - """ - The version of the explode operation, either "array" or "string". - """ - return self._version - - @property - def delimiter(self) -> str | None: - """ - The delimiter to use when exploding a string column, or None if the - version is "array". - """ - return self._delimiter - - @property - def filtering(self) -> bool: - """ - Whether the explode operation can result in not every row from the - original being included in the output, i.e. if some of the rows from - `data` are empty arrays or empty strings. - """ - return self._filtering - - @property - def is_distinct(self) -> bool: - """ - Whether each exploded value will be unique within the output with - regards to the original row from which it was exploded. - """ - return self._is_distinct + return self._explode_spec @property def calc_terms(self) -> set[str]: - if self._index_name is None: - return {self._value_name} + if self.explode_spec.index_name is None: + return {self.explode_spec.value_name} else: - return {self._value_name, self._index_name} + return {self.explode_spec.value_name, self.explode_spec.index_name} @property def all_terms(self) -> set[str]: @@ -171,19 +121,20 @@ def ordering(self) -> list[CollationExpression] | None: @property def unique_terms(self) -> list[str]: - if self.is_distinct: - return [self.value_name] + # Note: must add ancestral unique terms in the hybrid step + if self.explode_spec.is_distinct: + return [self.explode_spec.value_name] else: - assert self.index_name is not None - return [self.index_name] # Note: must add ancestral unique terms + assert self.explode_spec.index_name is not None + return [self.explode_spec.index_name] def is_singular(self, context: PyDoughCollectionQDAG) -> bool: return False def get_expression_position(self, expr_name: str) -> int: - if expr_name == self._value_name: + if expr_name == self.explode_spec.value_name: return 0 - elif expr_name == self._index_name: + elif expr_name == self.explode_spec.index_name: return 1 else: raise PyDoughQDAGException(f"Unrecognized term of {self!r}: {expr_name!r}") @@ -232,13 +183,13 @@ def get_term(self, term_name: str) -> PyDoughQDAG: ) typ: PyDoughType - if term_name == self._value_name: + if term_name == self.explode_spec.value_name: if isinstance(self._data.pydough_type, ArrayType): typ = self._data.pydough_type.elem_type else: typ = UnknownType() else: - assert term_name == self._index_name + assert term_name == self.explode_spec.index_name typ = NumericType() return Reference(self, term_name, typ) @@ -249,17 +200,10 @@ def to_string(self) -> str: @property def standalone_string(self) -> str: terms: list[str] = [ - f"EXPLODE[{self._data.to_string()}", - f"name={self._name!r}", - f"value_name={self._value_name!r}", + f"EXPLODE[{self.data.to_string()}", + f"name={self.name!r}", + self.explode_spec.keyword_arg_string, ] - if self._index_name is not None: - terms.append(f"index_name={self._index_name!r}") - terms.append(f"version={self._version!r}") - if self._version == "string": - terms.append(f"delimiter={self._delimiter!r}") - terms.append(f"filtering={self._filtering}") - terms.append(f"is_distinct={self._is_distinct})") return ", ".join(terms) @property @@ -272,10 +216,6 @@ def equals(self, other: object) -> bool: isinstance(other, Explode) and super().equals(other) and self.data == other.data - and self.value_name == other.value_name - and self.index_name == other.index_name - and self.version == other.version - and self.delimiter == other.delimiter - and self.filtering == other.filtering - and self.is_distinct == other.is_distinct + and self.name == other.name + and self.explode_spec == other.explode_spec ) diff --git a/pydough/qdag/node_builder.py b/pydough/qdag/node_builder.py index 812257793..9fea86f81 100644 --- a/pydough/qdag/node_builder.py +++ b/pydough/qdag/node_builder.py @@ -22,6 +22,7 @@ ) from pydough.types import PyDoughType from pydough.user_collections.user_collections import PyDoughUserGeneratedCollection +from pydough.utilities import ExplodeSpec from .abstract_pydough_qdag import PyDoughQDAG from .collections import ( @@ -407,12 +408,7 @@ def build_explode( preceding_context: PyDoughCollectionQDAG, data: PyDoughExpressionQDAG, name: str, - value_name: str, - index_name: str | None, - version: str, - delimiter: str | None, - filtering: bool, - is_distinct: bool, + explode_spec: ExplodeSpec, ): """ Creates an EXPLODE instance. @@ -421,18 +417,7 @@ def build_explode( `preceding_context`: the preceding collection. `data`: the data to be exploded. `name`: the name of the collection after being exploded. - `value_name`: the name of the value column in the exploded - collection. - `index_name`: the name of the index column in the exploded - collection. - `version`: the version of the explode operation (e.g., "string" or - "array"). - `delimiter`: the delimiter used for string explosion (if - applicable). - `filtering`: whether the explode operation is filtering (i.e., some - rows may be dropped). - `is_distinct`: whether the exploded values are distinct with regards - to the original row. + `explode_spec`: the specification of the explode operation. Returns: The newly created PyDough EXPLODE instance. @@ -441,12 +426,7 @@ def build_explode( preceding_context, data, name, - value_name, - index_name, - version, - delimiter, - filtering, - is_distinct, + explode_spec, ) def build_generated_collection( diff --git a/pydough/unqualified/qualification.py b/pydough/unqualified/qualification.py index 8efa34c2f..162e7793c 100644 --- a/pydough/unqualified/qualification.py +++ b/pydough/unqualified/qualification.py @@ -35,6 +35,7 @@ WindowCall, ) from pydough.types import PyDoughType +from pydough.utilities import ExplodeSpec from .unqualified_node import ( UnqualifiedAccess, @@ -1296,12 +1297,7 @@ def qualify_explode( unqualified_parent: UnqualifiedNode = unqualified._parcel[0] data_raw: UnqualifiedNode = unqualified._parcel[1] name: str = unqualified._parcel[2] - value_name: str = unqualified._parcel[3] - index_name: str | None = unqualified._parcel[4] - version: str = unqualified._parcel[5] - delimiter: str | None = unqualified._parcel[6] - filtering: bool = unqualified._parcel[7] - is_distinct: bool = unqualified._parcel[8] + explode_spec: ExplodeSpec = unqualified._parcel[3] qualified_parent: PyDoughCollectionQDAG = self.qualify_collection( unqualified_parent, context, is_child @@ -1320,12 +1316,7 @@ def qualify_explode( qualified_parent, qualified_data, name, - value_name, - index_name, - version, - delimiter, - filtering, - is_distinct, + explode_spec, ) if isinstance(unqualified_parent, UnqualifiedRoot) and is_child: answer = ChildOperatorChildAccess(answer) diff --git a/pydough/unqualified/unqualified_node.py b/pydough/unqualified/unqualified_node.py index 4225aade4..94d6909de 100644 --- a/pydough/unqualified/unqualified_node.py +++ b/pydough/unqualified/unqualified_node.py @@ -42,6 +42,7 @@ UnknownType, ) from pydough.user_collections.user_collections import PyDoughUserGeneratedCollection +from pydough.utilities import ExplodeSpec class UnqualifiedNode(ABC): @@ -519,12 +520,9 @@ def EXPLODE( self, data, name, - value_name, - index_name, - version, - delimiter, - filtering, - is_distinct, + ExplodeSpec( + value_name, index_name, version, delimiter, filtering, is_distinct + ), ) @@ -895,33 +893,18 @@ def __init__( predecessor: UnqualifiedNode, data: UnqualifiedNode, name: str, - value_name: str, - index_name: str | None, - version: str, - delimiter: str | None, - filtering: bool, - is_distinct: bool, + explode_spec: ExplodeSpec, ): self._parcel: tuple[ UnqualifiedNode, UnqualifiedNode, str, - str, - str | None, - str, - str | None, - bool, - bool, + ExplodeSpec, ] = ( predecessor, data, name, - value_name, - index_name, - version, - delimiter, - filtering, - is_distinct, + explode_spec, ) @@ -1032,14 +1015,7 @@ def display_raw(unqualified: UnqualifiedNode) -> str: result = f"{display_raw(unqualified._parcel[0])}.EXPLODE(" result += display_raw(unqualified._parcel[1]) result += f", name={unqualified._parcel[2]!r}" - result += f", value_name={unqualified._parcel[3]!r}" - if unqualified._parcel[4] is not None: - result += f", index_name={unqualified._parcel[4]!r}" - result += f", version={unqualified._parcel[5]!r}" - if unqualified._parcel[5] == "string": - result += f", delimiter={unqualified._parcel[6]!r}" - result += f", filtering={unqualified._parcel[7]}" - result += f", is_distinct={unqualified._parcel[8]}" + result += f", {unqualified._parcel[3].keyword_arg_string}" return result + ")" case _: raise PyDoughUnqualifiedException( diff --git a/pydough/utilities/__init__.py b/pydough/utilities/__init__.py new file mode 100644 index 000000000..ae3232c03 --- /dev/null +++ b/pydough/utilities/__init__.py @@ -0,0 +1,8 @@ +""" +Module of PyDough dealing with the definitions of various utilities and +specifications that can be accessed throughout PyDough. +""" + +__all__ = ["ExplodeSpec"] + +from .explode_spec import ExplodeSpec diff --git a/pydough/utilities/explode_spec.py b/pydough/utilities/explode_spec.py new file mode 100644 index 000000000..3470ff3a7 --- /dev/null +++ b/pydough/utilities/explode_spec.py @@ -0,0 +1,45 @@ +""" +Contains the definition of the ExplodeSpec dataclass, +""" + +from dataclasses import dataclass + + +@dataclass +class ExplodeSpec: + """ + Dataclass storing information about an EXPLODE operation being performed. + """ + + value_name: str + index_name: str | None + version: str + delimiter: str | None + filtering: bool + is_distinct: bool + + @property + def arg_list_string(self): + args: list[str] = [] + args.append(self.value_name) + if self.index_name is not None: + args.append(self.index_name) + args.append(self.version) + if self.delimiter is not None: + args.append(self.delimiter) + args.append(repr(self.filtering)) + args.append(repr(self.is_distinct)) + return ", ".join(args) + + @property + def keyword_arg_string(self): + kwargs: list[str] = [] + kwargs.append(f"value_name={self.value_name!r}") + if self.index_name is not None: + kwargs.append(f"index_name={self.index_name!r}") + kwargs.append(f"version={self.version!r}") + if self.delimiter is not None: + kwargs.append(f"delimiter={self.delimiter!r}") + kwargs.append(f"filtering={self.filtering!r}") + kwargs.append(f"is_distinct={self.is_distinct!r}") + return ", ".join(kwargs) From d0473e5d691479380c5a76478c27796061bdd806 Mon Sep 17 00:00:00 2001 From: knassre-bodo Date: Mon, 27 Jul 2026 10:15:23 -0700 Subject: [PATCH 16/44] WIP Relational steps --- pydough/conversion/hybrid_expressions.py | 2 ++ pydough/conversion/relational_converter.py | 29 +++++++++++++++++-- .../conversion/relational_simplification.py | 8 +++++ .../relational/relational_nodes/explode.py | 25 +++++++++------- .../relational_expression_dispatcher.py | 5 ++++ ...elational_expression_shuttle_dispatcher.py | 11 +++++++ .../relational_nodes/relational_visitor.py | 13 ++++++++- .../relational_nodes/tree_string_visitor.py | 7 +++-- pydough/sqlglot/sqlglot_relational_visitor.py | 4 +++ 9 files changed, 89 insertions(+), 15 deletions(-) diff --git a/pydough/conversion/hybrid_expressions.py b/pydough/conversion/hybrid_expressions.py index 8b8f19843..245dcf917 100644 --- a/pydough/conversion/hybrid_expressions.py +++ b/pydough/conversion/hybrid_expressions.py @@ -325,6 +325,8 @@ def to_string(self): def shift_back(self, levels: int, shift_correl: bool = True) -> HybridExpr: if levels == 0: return self + elif levels < 0 and abs(levels) == self.back_idx: + return HybridRefExpr(self.name, self.typ) return HybridBackRefExpr(self.name, self.back_idx + levels, self.typ) def squish_backrefs_into_correl( diff --git a/pydough/conversion/relational_converter.py b/pydough/conversion/relational_converter.py index e3a5884e7..78ec9600c 100644 --- a/pydough/conversion/relational_converter.py +++ b/pydough/conversion/relational_converter.py @@ -1410,9 +1410,31 @@ def translate_explode( """ TODO """ - raise NotImplementedError() - new_node = Explode() + exploded_data: RelationalExpression = self.translate_expression( + operation.explode_data.shift_back(-1), context + ) + input_mapping: dict[str, RelationalExpression] = { + name: ColumnReference(name, expr.data_type) + for name, expr in context.relational_node.columns.items() + } + new_node: RelationalNode = Explode( + context.relational_node, + exploded_data, + operation.explode_spec, + input_mapping, + ) new_expressions: dict[HybridExpr, ColumnReference] = {} + for hybrid_expr, rel_expr in context.expressions.items(): + new_expressions[hybrid_expr.shift_back(1)] = rel_expr + new_expressions[ + HybridRefExpr(operation.explode_spec.value_name, operation.explode_data.typ) + ] = ColumnReference( + operation.explode_spec.value_name, operation.explode_data.typ + ) + if operation.explode_spec.index_name is not None: + new_expressions[ + HybridRefExpr(operation.explode_spec.index_name, NumericType()) + ] = ColumnReference(operation.explode_spec.index_name, NumericType()) return TranslationOutput(new_node, new_expressions) def translate_hybridroot(self, context: TranslationOutput) -> TranslationOutput: @@ -1912,6 +1934,9 @@ def convert_ast_to_relational( hybrid, len(hybrid.pipeline) - 1 ) + print() + print(output.relational_node.to_tree_string()) + # Extract the relevant expressions for the final columns and ordering keys # so that the root node can be built from them. raw_result: RelationalRoot = postprocess_root(node, columns, hybrid, output) diff --git a/pydough/conversion/relational_simplification.py b/pydough/conversion/relational_simplification.py index 63c5a72ce..2eccb5b8a 100644 --- a/pydough/conversion/relational_simplification.py +++ b/pydough/conversion/relational_simplification.py @@ -23,6 +23,7 @@ ColumnReference, CorrelatedReference, EmptySingleton, + Explode, Filter, GeneratedTable, Join, @@ -1649,6 +1650,13 @@ def visit_project(self, node: Project) -> None: ) self.stack.append(output_predicates) + def visit_explode(self, node: Explode) -> None: + output_predicates: dict[RelationalExpression, PredicateSet] = ( + self.generic_visit(node) + ) + node._explode_data = node._explode_data.accept_shuttle(self.shuttle) + self.stack.append(output_predicates) + def infer_null_predicates_from_condition( self, output_predicates: dict[RelationalExpression, PredicateSet], diff --git a/pydough/relational/relational_nodes/explode.py b/pydough/relational/relational_nodes/explode.py index 7790384ed..042058b8a 100644 --- a/pydough/relational/relational_nodes/explode.py +++ b/pydough/relational/relational_nodes/explode.py @@ -7,6 +7,7 @@ from typing import TYPE_CHECKING from pydough.relational.relational_expressions import RelationalExpression +from pydough.utilities import ExplodeSpec from .abstract_node import RelationalNode from .single_relational import SingleRelational @@ -25,16 +26,12 @@ def __init__( self, input: RelationalNode, explode_data: RelationalExpression, - value_name: str, - index_name: str | None, - version: str, - delimiter: str | None, - filtering: bool, - is_distinct: bool, + explode_spec: ExplodeSpec, columns: dict[str, RelationalExpression], ) -> None: super().__init__(input, columns) self._explode_data: RelationalExpression = explode_data + self._explode_spec: ExplodeSpec = explode_spec @property def explode_data(self) -> RelationalExpression: @@ -43,19 +40,26 @@ def explode_data(self) -> RelationalExpression: """ return self._explode_data + @property + def explode_spec(self) -> ExplodeSpec: + """ + The specification of the explode operation. + """ + return self._explode_spec + def node_equals(self, other: RelationalNode) -> bool: return ( isinstance(other, Explode) and self.explode_data == other.explode_data + and self.explode_spec == other.explode_spec and super().node_equals(other) ) def to_string(self, compact: bool = False) -> str: - return "Explode(...)" + return f"Explode({self.explode_data.to_string(compact)}, {self.explode_spec.keyword_arg_string}, columns={self.make_column_string(self.columns, compact)})" def accept(self, visitor: "RelationalVisitor") -> None: - raise NotImplementedError() - visitor.visit_filter(self) + visitor.visit_explode(self) def accept_shuttle(self, shuttle: "RelationalShuttle") -> RelationalNode: raise NotImplementedError() @@ -65,4 +69,5 @@ def node_copy( columns: dict[str, RelationalExpression], inputs: list[RelationalNode], ) -> RelationalNode: - raise NotImplementedError() + assert len(inputs) == 1, "Explode node should have exactly one input" + return Explode(inputs[0], self.explode_data, self.explode_spec, columns) diff --git a/pydough/relational/relational_nodes/relational_expression_dispatcher.py b/pydough/relational/relational_nodes/relational_expression_dispatcher.py index e5ad70ba8..ec09e27c6 100644 --- a/pydough/relational/relational_nodes/relational_expression_dispatcher.py +++ b/pydough/relational/relational_nodes/relational_expression_dispatcher.py @@ -10,6 +10,7 @@ from .abstract_node import RelationalNode from .aggregate import Aggregate from .empty_singleton import EmptySingleton +from .explode import Explode from .filter import Filter from .join import Join from .limit import Limit @@ -80,3 +81,7 @@ def visit_root(self, root: RelationalRoot) -> None: def visit_generated_table(self, generated_table) -> None: self.visit_common(generated_table) + + def visit_explode(self, explode: Explode): + self.visit_common(explode) + explode.explode_data.accept(self._expr_visitor) diff --git a/pydough/relational/relational_nodes/relational_expression_shuttle_dispatcher.py b/pydough/relational/relational_nodes/relational_expression_shuttle_dispatcher.py index 1602c81cc..7340b9907 100644 --- a/pydough/relational/relational_nodes/relational_expression_shuttle_dispatcher.py +++ b/pydough/relational/relational_nodes/relational_expression_shuttle_dispatcher.py @@ -11,6 +11,7 @@ from .abstract_node import RelationalNode from .aggregate import Aggregate from .empty_singleton import EmptySingleton +from .explode import Explode from .filter import Filter from .generated_table import GeneratedTable from .join import Join @@ -78,6 +79,16 @@ def visit_empty_singleton(self, singleton: EmptySingleton) -> None: def visit_generated_table(self, generated_table: GeneratedTable) -> None: pass + def visit_explode(self, explode: Explode) -> None: + self.visit_inputs(explode) + explode._explode_data = explode._explode_data.accept_shuttle(self.shuttle) + for name, expr in explode.columns.items(): + if name not in ( + explode.explode_spec.value_name, + explode.explode_spec.index_name, + ): + explode.columns[name] = expr.accept_shuttle(self.shuttle) + def visit_root(self, root: RelationalRoot) -> None: self.visit_common(root) if root.limit is not None: diff --git a/pydough/relational/relational_nodes/relational_visitor.py b/pydough/relational/relational_nodes/relational_visitor.py index 7d5530383..3a8a8bf78 100644 --- a/pydough/relational/relational_nodes/relational_visitor.py +++ b/pydough/relational/relational_nodes/relational_visitor.py @@ -11,7 +11,9 @@ from .abstract_node import RelationalNode from .aggregate import Aggregate from .empty_singleton import EmptySingleton +from .explode import Explode from .filter import Filter +from .generated_table import GeneratedTable from .join import Join from .limit import Limit from .project import Project @@ -121,10 +123,19 @@ def visit_root(self, root: RelationalRoot) -> None: """ @abstractmethod - def visit_generated_table(self, generated_table) -> None: + def visit_generated_table(self, generated_table: GeneratedTable) -> None: """ Visit a GeneratedTable node. Args: `generated_table`: The generated table node to visit. """ + + @abstractmethod + def visit_explode(self, explode: Explode) -> None: + """ + Visit an Explode node. + + Args: + `explode`: The explode node to visit. + """ diff --git a/pydough/relational/relational_nodes/tree_string_visitor.py b/pydough/relational/relational_nodes/tree_string_visitor.py index 3dc56eda3..f9205d3ee 100644 --- a/pydough/relational/relational_nodes/tree_string_visitor.py +++ b/pydough/relational/relational_nodes/tree_string_visitor.py @@ -63,5 +63,8 @@ def visit_empty_singleton(self, empty_singleton) -> None: def visit_root(self, root) -> None: self.visit_node(root) - def visit_generated_table(self, root) -> None: - self.visit_node(root) + def visit_generated_table(self, generated_table) -> None: + self.visit_node(generated_table) + + def visit_explode(self, explode): + self.visit_node(explode) diff --git a/pydough/sqlglot/sqlglot_relational_visitor.py b/pydough/sqlglot/sqlglot_relational_visitor.py index ae58224fb..b4846e3ec 100644 --- a/pydough/sqlglot/sqlglot_relational_visitor.py +++ b/pydough/sqlglot/sqlglot_relational_visitor.py @@ -29,6 +29,7 @@ ColumnReferenceInputNameModifier, CorrelatedReference, EmptySingleton, + Explode, ExpressionSortInfo, Filter, GeneratedTable, @@ -581,6 +582,9 @@ def visit_generated_table(self, generated_table: "GeneratedTable") -> None: ) self._stack.append(query) + def visit_explode(self, explode: Explode) -> None: + raise NotImplementedError() + def relational_to_sqlglot(self, root: RelationalRoot) -> SQLGlotExpression: """ Interface to convert an entire relational tree to a SQLGlot expression. From ac158cb3effef575747e9b0a496c179251b6638a Mon Sep 17 00:00:00 2001 From: knassre-bodo Date: Wed, 29 Jul 2026 10:20:08 -0700 Subject: [PATCH 17/44] Continued work on relational handling --- pydough/conversion/agg_removal.py | 28 +++++++++++++++++++ pydough/conversion/relational_converter.py | 2 ++ pydough/qdag/collections/explode.py | 6 ++-- .../relational/relational_nodes/explode.py | 2 +- .../relational_nodes/relational_shuttle.py | 10 +++++++ tests/test_pipeline_tpch_custom.py | 7 +++-- 6 files changed, 50 insertions(+), 5 deletions(-) diff --git a/pydough/conversion/agg_removal.py b/pydough/conversion/agg_removal.py index 1c13737b5..b24d43e2f 100644 --- a/pydough/conversion/agg_removal.py +++ b/pydough/conversion/agg_removal.py @@ -11,6 +11,7 @@ Aggregate, CallExpression, EmptySingleton, + Explode, Filter, GeneratedTable, Join, @@ -218,6 +219,29 @@ def deduce_join_uniqueness( return result +def deduce_explode_uniqueness( + unique_terms: set[frozenset[str]], explode: Explode +) -> set[frozenset[str]]: + """ + Helper function to transforms the uniqueness sets after an Explode + operation duplicates rows. + """ + # Build up the list of column names that imply uniqueness within one of the + # expanded row sets (either the index column, or the expanded data, or + # both) + cross_terms: list[str] = [] + if explode.explode_spec.index_name is not None: + cross_terms.append(explode.explode_spec.index_name) + if explode.explode_spec.is_distinct: + cross_terms.append(explode.explode_spec.value_name) + # Add each of the cross terms to all of the uniqueness sets + result: set[frozenset[str]] = set() + for term_set in unique_terms: + for cross_term in cross_terms: + result.add(term_set | frozenset([cross_term])) + return result + + def aggregation_uniqueness_helper( node: RelationalNode, ) -> tuple[RelationalNode, set[frozenset[str]]]: @@ -286,6 +310,10 @@ def aggregation_uniqueness_helper( node, unique_sets ) return node, final_uniqueness + case Explode(): + node._input, input_uniqueness = aggregation_uniqueness_helper(node.input) + input_uniqueness = bubble_uniqueness(input_uniqueness, node.columns, None) + return node, deduce_explode_uniqueness(input_uniqueness, node) # Empty singletons don't have uniqueness information. case EmptySingleton(): return node, set() diff --git a/pydough/conversion/relational_converter.py b/pydough/conversion/relational_converter.py index 78ec9600c..82fd9d4f7 100644 --- a/pydough/conversion/relational_converter.py +++ b/pydough/conversion/relational_converter.py @@ -1936,6 +1936,8 @@ def convert_ast_to_relational( print() print(output.relational_node.to_tree_string()) + print() + print(output.expressions) # Extract the relevant expressions for the final columns and ordering keys # so that the root node can be built from them. diff --git a/pydough/qdag/collections/explode.py b/pydough/qdag/collections/explode.py index 77f522d3c..56acf251b 100644 --- a/pydough/qdag/collections/explode.py +++ b/pydough/qdag/collections/explode.py @@ -140,6 +140,8 @@ def get_expression_position(self, expr_name: str) -> int: raise PyDoughQDAGException(f"Unrecognized term of {self!r}: {expr_name!r}") def get_term(self, term_name: str) -> PyDoughQDAG: + self.verify_term_exists(term_name) + if term_name not in self.all_terms: if term_name in self.ancestor_context.all_terms: result: PyDoughQDAG = self.ancestor_context.get_term(term_name) @@ -200,11 +202,11 @@ def to_string(self) -> str: @property def standalone_string(self) -> str: terms: list[str] = [ - f"EXPLODE[{self.data.to_string()}", + self.data.to_string(), f"name={self.name!r}", self.explode_spec.keyword_arg_string, ] - return ", ".join(terms) + return f"EXPLODE[{', '.join(terms)}]" @property def tree_item_string(self) -> str: diff --git a/pydough/relational/relational_nodes/explode.py b/pydough/relational/relational_nodes/explode.py index 042058b8a..ea0404a46 100644 --- a/pydough/relational/relational_nodes/explode.py +++ b/pydough/relational/relational_nodes/explode.py @@ -62,7 +62,7 @@ def accept(self, visitor: "RelationalVisitor") -> None: visitor.visit_explode(self) def accept_shuttle(self, shuttle: "RelationalShuttle") -> RelationalNode: - raise NotImplementedError() + return shuttle.visit_explode(self) def node_copy( self, diff --git a/pydough/relational/relational_nodes/relational_shuttle.py b/pydough/relational/relational_nodes/relational_shuttle.py index 2dadcca92..f6ecec2ee 100644 --- a/pydough/relational/relational_nodes/relational_shuttle.py +++ b/pydough/relational/relational_nodes/relational_shuttle.py @@ -9,6 +9,7 @@ from .abstract_node import RelationalNode from .aggregate import Aggregate from .empty_singleton import EmptySingleton +from .explode import Explode from .filter import Filter from .generated_table import GeneratedTable from .join import Join @@ -128,3 +129,12 @@ def visit_root(self, root: RelationalRoot) -> RelationalNode: `root`: The root node to visit. """ return self.generic_visit_inputs(root) + + def visit_explode(self, explode: Explode) -> RelationalNode: + """ + Visit an Explode node. + + Args: + `explode`: The Explode node to visit. + """ + return self.generic_visit_inputs(explode) diff --git a/tests/test_pipeline_tpch_custom.py b/tests/test_pipeline_tpch_custom.py index befe9d860..dc9f046a3 100644 --- a/tests/test_pipeline_tpch_custom.py +++ b/tests/test_pipeline_tpch_custom.py @@ -4321,7 +4321,9 @@ "array_data = pydough.dataframe_collection(name='tbl', dataframe=array_df, unique_column_names=['key'])\n" "result = (" " array_data" - " .EXPLODE(arr, 'exploded_array', index_name='arr_idx', value_name='arr_val', version='array'filtering=True, is_distinct=True)" + " .CALCULATE(key, arr)" + " .EXPLODE(arr, 'exploded_array', index_name='arr_idx', value_name='arr_val', version='array', filtering=True, is_distinct=True)" + " .CALCULATE(key, arr, arr_idx, arr_val)" " .ORDER_BY(key, arr_idx)" ")", "TPCH", @@ -4338,7 +4340,7 @@ [5, 6], ], "arr_idx": [1, 1, 2, 3, 4, 1, 2], - "nation_name": [1, 2, 3, None, 4, 5, 6], + "arr_val": [1, 2, 3, None, 4, 5, 6], } ), "explode_02", @@ -4359,6 +4361,7 @@ PyDoughPandasTest( "result = (" " customers" + " .CALCULATE(name)" " .TOP_K(5, by=key.ASC())" " .EXPLODE(name, 'exploded_customers', index_name='idx', value_name='val', version='string', delimiter='#')" " .ORDER_BY(name, idx)" From e72bb2f660f2a2004db7f86e95c5237028d09fa6 Mon Sep 17 00:00:00 2001 From: knassre-bodo Date: Fri, 31 Jul 2026 10:17:39 -0700 Subject: [PATCH 18/44] WIP relational handling and qdag collection access --- pydough/conversion/hybrid_translator.py | 14 ++-- pydough/conversion/relational_converter.py | 8 --- pydough/qdag/collections/explode.py | 38 +++++----- .../relational/relational_nodes/explode.py | 17 ++++- tests/test_pipeline_tpch_custom.py | 72 +++++++++++++++++++ tests/test_plan_refsols/explode_01.txt | 6 ++ tests/test_plan_refsols/explode_02.txt | 3 + tests/test_plan_refsols/explode_03.txt | 4 ++ tests/test_plan_refsols/explode_04.txt | 6 ++ .../simple_pydough_functions.py | 1 + tests/test_qualification.py | 18 ++--- 11 files changed, 139 insertions(+), 48 deletions(-) create mode 100644 tests/test_plan_refsols/explode_01.txt create mode 100644 tests/test_plan_refsols/explode_02.txt create mode 100644 tests/test_plan_refsols/explode_03.txt create mode 100644 tests/test_plan_refsols/explode_04.txt diff --git a/pydough/conversion/hybrid_translator.py b/pydough/conversion/hybrid_translator.py index b7c543e6a..af717e3ff 100644 --- a/pydough/conversion/hybrid_translator.py +++ b/pydough/conversion/hybrid_translator.py @@ -1242,7 +1242,10 @@ def process_hybrid_collations( name: str expr: HybridExpr for collation in collations: - if type(collation.expr) is Reference: + if ( + type(collation.expr) is Reference + and collation.expr.term_name in hybrid.pipeline[-1].terms + ): name = collation.expr.term_name else: name = self.get_ordering_name(hybrid) @@ -1782,14 +1785,8 @@ def convert_qdag_to_hybrid(self, node: PyDoughCollectionQDAG) -> HybridTree: The HybridTree representation of the given QDAG node after transformations. """ - print() - print(node.to_tree_string()) - print() # 1. Run the initial conversion from QDAG to Hybrid hybrid: HybridTree = self.make_hybrid_tree(node, None) - print() - print(hybrid) - print() # 2. Eject any aggregate inputs from the hybrid tree. self.eject_aggregate_inputs(hybrid) # 3. Syncretize any children of the hybrid tree that share a common @@ -1799,9 +1796,6 @@ def convert_qdag_to_hybrid(self, node: PyDoughCollectionQDAG) -> HybridTree: # filters with correlated references into join conditions. self.run_correlation_extraction(hybrid) # 5. Run the de-correlation procedure. - print() - print(hybrid) - print() self.run_hybrid_decorrelation(hybrid) # 6. Run the filter-merging procedure, then re-run ejecting aggregate # inputs to clean up any new aggregates created by filter merging. diff --git a/pydough/conversion/relational_converter.py b/pydough/conversion/relational_converter.py index 82fd9d4f7..16ddbb94f 100644 --- a/pydough/conversion/relational_converter.py +++ b/pydough/conversion/relational_converter.py @@ -1925,20 +1925,12 @@ def convert_ast_to_relational( hybrid_translator: HybridTranslator = HybridTranslator(session) hybrid: HybridTree = hybrid_translator.convert_qdag_to_hybrid(node) - print() - print(hybrid) - # Then, invoke relational conversion procedure. The first element in the # returned list is the final relational tree. output: TranslationOutput = rel_translator.rel_translation( hybrid, len(hybrid.pipeline) - 1 ) - print() - print(output.relational_node.to_tree_string()) - print() - print(output.expressions) - # Extract the relevant expressions for the final columns and ordering keys # so that the root node can be built from them. raw_result: RelationalRoot = postprocess_root(node, columns, hybrid, output) diff --git a/pydough/qdag/collections/explode.py b/pydough/qdag/collections/explode.py index 56acf251b..3260e9435 100644 --- a/pydough/qdag/collections/explode.py +++ b/pydough/qdag/collections/explode.py @@ -49,6 +49,12 @@ def __init__( self._data: PyDoughExpressionQDAG = data self._explode_spec: ExplodeSpec = explode_spec self._all_property_names: set[str] = set() + # Build a mapping of inherited subcollections from ancestor context. + self._inherited_subcollections: dict[str, PyDoughCollectionQDAG] = {} + for name in ancestor.all_terms: + term = ancestor.get_term(name) + if isinstance(term, PyDoughCollectionQDAG): + self._inherited_subcollections[name] = term # Build the current node's ancestral mapping by copying the ancestor's # mapping and incrementing each level by 1 to reflect # the added depth of this node. @@ -59,6 +65,7 @@ def __init__( self._all_property_names.add(explode_spec.value_name) if explode_spec.index_name is not None: self._all_property_names.add(explode_spec.index_name) + self._all_property_names.update(set(self._inherited_subcollections)) def clone_with_parent(self, new_parent: PyDoughCollectionQDAG) -> "Explode": return Explode( @@ -115,6 +122,14 @@ def ancestral_mapping(self) -> dict[str, int]: def inherited_downstreamed_terms(self) -> set[str]: return self.ancestor_context.inherited_downstreamed_terms + @property + def inherited_subcollections(self) -> dict[str, PyDoughCollectionQDAG]: + """ + All of the collection properties that the EXPLODE operator has + access to from its ancestor context. + """ + return self._inherited_subcollections + @property def ordering(self) -> list[CollationExpression] | None: return None @@ -142,22 +157,6 @@ def get_expression_position(self, expr_name: str) -> int: def get_term(self, term_name: str) -> PyDoughQDAG: self.verify_term_exists(term_name) - if term_name not in self.all_terms: - if term_name in self.ancestor_context.all_terms: - result: PyDoughQDAG = self.ancestor_context.get_term(term_name) - if isinstance(result, PyDoughExpressionQDAG): - if isinstance(result, BackReferenceExpression): - return BackReferenceExpression( - self, term_name, result.back_levels + 1 - ) - return BackReferenceExpression(self, term_name, 1) - else: - return result - else: - raise pydough.active_session.error_builder.term_not_found( - collection=self, term_name=term_name - ) - # Special handling of terms down-streamed from an ancestor CALCULATE # clause. if term_name in self.ancestral_mapping: @@ -184,6 +183,9 @@ def get_term(self, term_name: str) -> PyDoughQDAG: context, term_name, context.get_expr(term_name).pydough_type ) + if term_name in self.inherited_subcollections: + raise NotImplementedError() + typ: PyDoughType if term_name == self.explode_spec.value_name: if isinstance(self._data.pydough_type, ArrayType): @@ -206,12 +208,12 @@ def standalone_string(self) -> str: f"name={self.name!r}", self.explode_spec.keyword_arg_string, ] - return f"EXPLODE[{', '.join(terms)}]" + return f"Explode[{', '.join(terms)}]" @property def tree_item_string(self) -> str: base_str: str = self.standalone_string - return f"EXPLODE[{base_str[8:-1]}]" + return f"Explode[{base_str[8:-1]}]" def equals(self, other: object) -> bool: return ( diff --git a/pydough/relational/relational_nodes/explode.py b/pydough/relational/relational_nodes/explode.py index ea0404a46..fc3ae5ab1 100644 --- a/pydough/relational/relational_nodes/explode.py +++ b/pydough/relational/relational_nodes/explode.py @@ -6,7 +6,10 @@ from typing import TYPE_CHECKING -from pydough.relational.relational_expressions import RelationalExpression +from pydough.relational.relational_expressions import ( + ColumnReference, + RelationalExpression, +) from pydough.utilities import ExplodeSpec from .abstract_node import RelationalNode @@ -29,9 +32,17 @@ def __init__( explode_spec: ExplodeSpec, columns: dict[str, RelationalExpression], ) -> None: - super().__init__(input, columns) self._explode_data: RelationalExpression = explode_data self._explode_spec: ExplodeSpec = explode_spec + total_columns: dict[str, RelationalExpression] = {**columns} + total_columns[explode_spec.value_name] = ColumnReference( + explode_spec.value_name, explode_data.data_type + ) + if explode_spec.index_name is not None: + total_columns[explode_spec.index_name] = ColumnReference( + explode_spec.index_name, explode_data.data_type + ) + super().__init__(input, total_columns) @property def explode_data(self) -> RelationalExpression: @@ -56,7 +67,7 @@ def node_equals(self, other: RelationalNode) -> bool: ) def to_string(self, compact: bool = False) -> str: - return f"Explode({self.explode_data.to_string(compact)}, {self.explode_spec.keyword_arg_string}, columns={self.make_column_string(self.columns, compact)})" + return f"EXPLODE({self.explode_data.to_string(compact)}, {self.explode_spec.keyword_arg_string}, columns={self.make_column_string(self.columns, compact)})" def accept(self, visitor: "RelationalVisitor") -> None: visitor.visit_explode(self) diff --git a/tests/test_pipeline_tpch_custom.py b/tests/test_pipeline_tpch_custom.py index dc9f046a3..3a4923f15 100644 --- a/tests/test_pipeline_tpch_custom.py +++ b/tests/test_pipeline_tpch_custom.py @@ -4417,6 +4417,78 @@ ), id="explode_04", ), + pytest.param( + PyDoughPandasTest( + "exploded_data = (" + " EXPLODE(name, 'exploded_names', index_name='idx', value_name='val', version='string', delimiter='I')" + " .WHERE(~STARTSWITH(nation.name, val[:1]))" + ")\n" + "result = (" + " regions" + " .CALCULATE(region_name=name, n_chunks=COUNT(exploded_data))" + " .ORDER_BY(region_name)" + ")", + "TPCH", + lambda: pd.DataFrame( + { + "region_name": [ + "AFRICA", + "AMERICA", + "ASIA", + "EUROPE", + "MIDDLE EAST", + ], + "n_chunks": [2, 2, 2, 1, 1], + } + ), + "explode_05", + order_sensitive=True, + skipped_dialects={"ANSI", "SQLITE"}, + ), + id="explode_05", + ), + pytest.param( + PyDoughPandasTest( + "result = (" + " customers" + " .CALCULATE(customer_name=name)" + " .TOP_K(5, by=key.ASC())" + " .EXPLODE(name, 'exploded_customers', index_name='idx', value_name='val', version='string', delimiter='#')" + " .CALCULATE(idx, val)" + " .nation" + " .CALCULATE(customer_name, idx, val, nation_name=name)" + " .ORDER_BY(customer_name, idx)" + ")", + "TPCH", + lambda: pd.DataFrame( + { + "customer_name": ["Customer#000000001"] * 2 + + ["Customer#000000002"] * 2 + + ["Customer#000000003"] * 2 + + ["Customer#000000004"] * 2 + + ["Customer#000000005"] * 2, + "idx": [1, 2] * 5, + "val": [ + "Customer", + "000000001", + "Customer", + "000000002", + "Customer", + "000000003", + "Customer", + "000000004", + "Customer", + "000000005", + ], + "nation_name": [""] * 10, + } + ), + "explode_06", + order_sensitive=True, + skipped_dialects={"ANSI", "SQLITE"}, + ), + id="explode_06", + ), pytest.param( PyDoughPandasTest( simple_range_1, diff --git a/tests/test_plan_refsols/explode_01.txt b/tests/test_plan_refsols/explode_01.txt new file mode 100644 index 000000000..45f9c6095 --- /dev/null +++ b/tests/test_plan_refsols/explode_01.txt @@ -0,0 +1,6 @@ +ROOT(columns=[('nation_name', nation_name), ('nation_idx', nation_idx)], orderings=[(region_name):asc_first, (nation_idx):asc_first]) + EXPLODE(nation_names, value_name='nation_name', index_name='nation_idx', version='array', filtering=False, is_distinct=True, columns={'nation_idx': nation_idx, 'nation_name': nation_name, 'region_name': region_name}) + JOIN(condition=t0.key == t1.region_key, type=INNER, cardinality=SINGULAR_ACCESS, reverse_cardinality=SINGULAR_ACCESS, columns={'nation_names': t1.nation_names, 'region_name': t0.region_name}) + SCAN(table=tpch.REGION, columns={'key': r_regionkey, 'region_name': r_name}) + AGGREGATE(keys={'region_key': region_key}, aggregations={'nation_names': LISTOF(name)}) + SCAN(table=tpch.NATION, columns={'name': n_name, 'region_key': n_regionkey}) diff --git a/tests/test_plan_refsols/explode_02.txt b/tests/test_plan_refsols/explode_02.txt new file mode 100644 index 000000000..2f0979614 --- /dev/null +++ b/tests/test_plan_refsols/explode_02.txt @@ -0,0 +1,3 @@ +ROOT(columns=[('key', key), ('arr', arr), ('arr_idx', arr_idx), ('arr_val', arr_val)], orderings=[(key):asc_first, (arr_idx):asc_first]) + EXPLODE(arr, value_name='arr_val', index_name='arr_idx', version='array', filtering=True, is_distinct=True, columns={'arr': arr, 'arr_idx': arr_idx, 'arr_val': arr_val, 'key': key}) + GENERATED_TABLE(DataframeCollection(name='tbl', shape=(4, 2), columns=['key', 'arr'])) diff --git a/tests/test_plan_refsols/explode_03.txt b/tests/test_plan_refsols/explode_03.txt new file mode 100644 index 000000000..2e56db35c --- /dev/null +++ b/tests/test_plan_refsols/explode_03.txt @@ -0,0 +1,4 @@ +ROOT(columns=[('val', val), ('idx', idx)], orderings=[(name):asc_first, (idx):asc_first]) + EXPLODE(name, value_name='val', index_name='idx', version='string', delimiter='#', filtering=True, is_distinct=False, columns={'idx': idx, 'name': name, 'val': val}) + LIMIT(limit=5:numeric, columns={'name': name}, orderings=[(key):asc_first]) + SCAN(table=tpch.CUSTOMER, columns={'key': c_custkey, 'name': c_name}) diff --git a/tests/test_plan_refsols/explode_04.txt b/tests/test_plan_refsols/explode_04.txt new file mode 100644 index 000000000..7ee25fa1f --- /dev/null +++ b/tests/test_plan_refsols/explode_04.txt @@ -0,0 +1,6 @@ +ROOT(columns=[('region_name', r_name), ('n_chunks', DEFAULT_TO(n_rows, 0:numeric))], orderings=[(r_name):asc_first]) + JOIN(condition=t0.r_regionkey == t1.key, type=LEFT, cardinality=SINGULAR_FILTER, reverse_cardinality=SINGULAR_ACCESS, columns={'n_rows': t1.n_rows, 'r_name': t0.r_name}) + SCAN(table=tpch.REGION, columns={'r_name': r_name, 'r_regionkey': r_regionkey}) + AGGREGATE(keys={'key': key}, aggregations={'n_rows': COUNT()}) + EXPLODE(name, value_name='val', index_name='idx', version='string', delimiter='I', filtering=True, is_distinct=False, columns={'idx': idx, 'key': key, 'val': val}) + SCAN(table=tpch.REGION, columns={'key': r_regionkey, 'name': r_name}) diff --git a/tests/test_pydough_functions/simple_pydough_functions.py b/tests/test_pydough_functions/simple_pydough_functions.py index 477d767c1..570a2c74a 100644 --- a/tests/test_pydough_functions/simple_pydough_functions.py +++ b/tests/test_pydough_functions/simple_pydough_functions.py @@ -160,6 +160,7 @@ def good_explode_08(): def good_explode_09(): + # TODO: handle the case of explode -> subcollection of original. return ( nations.CALCULATE(region_name=region.name) .EXPLODE( diff --git a/tests/test_qualification.py b/tests/test_qualification.py index 5825bc0d1..78ddc5758 100644 --- a/tests/test_qualification.py +++ b/tests/test_qualification.py @@ -973,7 +973,7 @@ """ ──┬─ TPCH └─┬─ TableCollection[nations] - └─── EXPLODE[name, name='exploded_nation', value_name='v', index_name='i', version='array', filtering=True, is_distinct=False] + └─── Explode[name, name='exploded_nation', value_name='v', index_name='i', version='array', filtering=True, is_distinct=False] """, id="good_explode_01", ), @@ -982,7 +982,7 @@ """ ──┬─ TPCH └─┬─ TableCollection[nations] - └─── EXPLODE[name, name='exploded_nation', value_name='v', index_name='i', version='array', filtering=True, is_distinct=True] + └─── Explode[name, name='exploded_nation', value_name='v', index_name='i', version='array', filtering=True, is_distinct=True] """, id="good_explode_02", ), @@ -991,7 +991,7 @@ """ ──┬─ TPCH └─┬─ TableCollection[nations] - └─── EXPLODE[name, name='exploded_nation', value_name='v', index_name='i', version='array', filtering=False, is_distinct=False] + └─── Explode[name, name='exploded_nation', value_name='v', index_name='i', version='array', filtering=False, is_distinct=False] """, id="good_explode_03", ), @@ -1000,7 +1000,7 @@ """ ──┬─ TPCH └─┬─ TableCollection[nations] - └─── EXPLODE[name, name='exploded_nation', value_name='v', index_name='i', version='array', filtering=True, is_distinct=False] + └─── Explode[name, name='exploded_nation', value_name='v', index_name='i', version='array', filtering=True, is_distinct=False] """, id="good_explode_04", ), @@ -1009,7 +1009,7 @@ """ ──┬─ TPCH └─┬─ TableCollection[nations] - └─── EXPLODE[name, name='exploded_nation', value_name='v', index_name='i', version='string', delimiter=' ', filtering=True, is_distinct=False] + └─── Explode[name, name='exploded_nation', value_name='v', index_name='i', version='string', delimiter=' ', filtering=True, is_distinct=False] """, id="good_explode_05", ), @@ -1018,7 +1018,7 @@ """ ──┬─ TPCH └─┬─ TableCollection[nations] - └─── EXPLODE[name, name='exploded_nation', value_name='v', index_name='i', version='string', delimiter=' ', filtering=True, is_distinct=True] + └─── Explode[name, name='exploded_nation', value_name='v', index_name='i', version='string', delimiter=' ', filtering=True, is_distinct=True] """, id="good_explode_06", ), @@ -1027,7 +1027,7 @@ """ ──┬─ TPCH └─┬─ TableCollection[nations] - └─── EXPLODE[name, name='exploded_nation', value_name='v', index_name='i', version='string', delimiter=' ', filtering=False, is_distinct=False] + └─── Explode[name, name='exploded_nation', value_name='v', index_name='i', version='string', delimiter=' ', filtering=False, is_distinct=False] """, id="good_explode_07", ), @@ -1036,7 +1036,7 @@ """ ──┬─ TPCH └─┬─ TableCollection[nations] - └─── EXPLODE[name, name='exploded_nation', value_name='v', index_name='i', version='string', delimiter=' ', filtering=True, is_distinct=False] + └─── Explode[name, name='exploded_nation', value_name='v', index_name='i', version='string', delimiter=' ', filtering=True, is_distinct=False] """, id="good_explode_08", ), @@ -1048,7 +1048,7 @@ └─┬─ Calculate[region_name=$1.name] ├─┬─ AccessChild │ └─── SubCollection[region] - ├─── EXPLODE[name, name='exploded_nation', value_name='v', index_name='i', version='string', delimiter=' ', filtering=True, is_distinct=False] + ├─── Explode[name, name='exploded_nation', value_name='v', index_name='i', version='string', delimiter=' ', filtering=True, is_distinct=False] └─┬─ Where[SLICE(v, None, 1, None) != SLICE(region_name, None, 1, None)] └─── SubCollection[customers] """, From 10ac9d9fba5958fbf36f832183f4c283c3dd6655 Mon Sep 17 00:00:00 2001 From: knassre-bodo Date: Mon, 3 Aug 2026 10:02:56 -0700 Subject: [PATCH 19/44] Skipped accessing sub-collections, for now --- pydough/qdag/collections/explode.py | 4 +++- tests/test_pipeline_tpch_custom.py | 8 +++++++- tests/test_pydough_functions/simple_pydough_functions.py | 1 - tests/test_qualification.py | 3 +++ 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/pydough/qdag/collections/explode.py b/pydough/qdag/collections/explode.py index 3260e9435..7bf0bc370 100644 --- a/pydough/qdag/collections/explode.py +++ b/pydough/qdag/collections/explode.py @@ -184,7 +184,9 @@ def get_term(self, term_name: str) -> PyDoughQDAG: ) if term_name in self.inherited_subcollections: - raise NotImplementedError() + raise PyDoughQDAGException( + "PyDough does not currently support accessing subcollections from an EXPLODE operator." + ) typ: PyDoughType if term_name == self.explode_spec.value_name: diff --git a/tests/test_pipeline_tpch_custom.py b/tests/test_pipeline_tpch_custom.py index 3a4923f15..dfb92f1a9 100644 --- a/tests/test_pipeline_tpch_custom.py +++ b/tests/test_pipeline_tpch_custom.py @@ -4421,7 +4421,7 @@ PyDoughPandasTest( "exploded_data = (" " EXPLODE(name, 'exploded_names', index_name='idx', value_name='val', version='string', delimiter='I')" - " .WHERE(~STARTSWITH(nation.name, val[:1]))" + " .WHERE(HAS(nations, STARTSWITH(name, val[:1])))" ")\n" "result = (" " regions" @@ -4446,6 +4446,9 @@ skipped_dialects={"ANSI", "SQLITE"}, ), id="explode_05", + marks=pytest.mark.skip( + "Skipping until PyDough supports accessing subcollections from an EXPLODE operator." + ), ), pytest.param( PyDoughPandasTest( @@ -4488,6 +4491,9 @@ skipped_dialects={"ANSI", "SQLITE"}, ), id="explode_06", + marks=pytest.mark.skip( + "Skipping until PyDough supports accessing subcollections from an EXPLODE operator." + ), ), pytest.param( PyDoughPandasTest( diff --git a/tests/test_pydough_functions/simple_pydough_functions.py b/tests/test_pydough_functions/simple_pydough_functions.py index 570a2c74a..477d767c1 100644 --- a/tests/test_pydough_functions/simple_pydough_functions.py +++ b/tests/test_pydough_functions/simple_pydough_functions.py @@ -160,7 +160,6 @@ def good_explode_08(): def good_explode_09(): - # TODO: handle the case of explode -> subcollection of original. return ( nations.CALCULATE(region_name=region.name) .EXPLODE( diff --git a/tests/test_qualification.py b/tests/test_qualification.py index 78ddc5758..e855c5f25 100644 --- a/tests/test_qualification.py +++ b/tests/test_qualification.py @@ -1053,6 +1053,9 @@ └─── SubCollection[customers] """, id="good_explode_09", + marks=pytest.mark.skip( + "Skipping until PyDough supports accessing subcollections from an EXPLODE operator." + ), ), ], ) From df4308eaf8943c027da9bbbf952bc2ad9c5f53c1 Mon Sep 17 00:00:00 2001 From: knassre-bodo Date: Wed, 5 Aug 2026 10:15:05 -0700 Subject: [PATCH 20/44] Minor testing patch --- pydough/qdag/collections/explode.py | 27 ++--- tests/test_pipeline_tpch_custom.py | 138 ++++++++++++++++++++++++- tests/test_plan_refsols/explode_07.txt | 7 ++ tests/test_qualification.py | 2 +- 4 files changed, 149 insertions(+), 25 deletions(-) create mode 100644 tests/test_plan_refsols/explode_07.txt diff --git a/pydough/qdag/collections/explode.py b/pydough/qdag/collections/explode.py index 7bf0bc370..3252e89c7 100644 --- a/pydough/qdag/collections/explode.py +++ b/pydough/qdag/collections/explode.py @@ -49,12 +49,6 @@ def __init__( self._data: PyDoughExpressionQDAG = data self._explode_spec: ExplodeSpec = explode_spec self._all_property_names: set[str] = set() - # Build a mapping of inherited subcollections from ancestor context. - self._inherited_subcollections: dict[str, PyDoughCollectionQDAG] = {} - for name in ancestor.all_terms: - term = ancestor.get_term(name) - if isinstance(term, PyDoughCollectionQDAG): - self._inherited_subcollections[name] = term # Build the current node's ancestral mapping by copying the ancestor's # mapping and incrementing each level by 1 to reflect # the added depth of this node. @@ -63,9 +57,10 @@ def __init__( } self._all_property_names.update(self._ancestral_mapping) self._all_property_names.add(explode_spec.value_name) + self._ancestral_mapping[explode_spec.value_name] = 0 if explode_spec.index_name is not None: self._all_property_names.add(explode_spec.index_name) - self._all_property_names.update(set(self._inherited_subcollections)) + self._ancestral_mapping[explode_spec.index_name] = 0 def clone_with_parent(self, new_parent: PyDoughCollectionQDAG) -> "Explode": return Explode( @@ -122,14 +117,6 @@ def ancestral_mapping(self) -> dict[str, int]: def inherited_downstreamed_terms(self) -> set[str]: return self.ancestor_context.inherited_downstreamed_terms - @property - def inherited_subcollections(self) -> dict[str, PyDoughCollectionQDAG]: - """ - All of the collection properties that the EXPLODE operator has - access to from its ancestor context. - """ - return self._inherited_subcollections - @property def ordering(self) -> list[CollationExpression] | None: return None @@ -159,7 +146,10 @@ def get_term(self, term_name: str) -> PyDoughQDAG: # Special handling of terms down-streamed from an ancestor CALCULATE # clause. - if term_name in self.ancestral_mapping: + if ( + term_name in self.ancestral_mapping + and self.ancestral_mapping[term_name] > 0 + ): # Verify that the ancestor name is not also a name in the current # context. if term_name in self.calc_terms: @@ -183,11 +173,6 @@ def get_term(self, term_name: str) -> PyDoughQDAG: context, term_name, context.get_expr(term_name).pydough_type ) - if term_name in self.inherited_subcollections: - raise PyDoughQDAGException( - "PyDough does not currently support accessing subcollections from an EXPLODE operator." - ) - typ: PyDoughType if term_name == self.explode_spec.value_name: if isinstance(self._data.pydough_type, ArrayType): diff --git a/tests/test_pipeline_tpch_custom.py b/tests/test_pipeline_tpch_custom.py index dfb92f1a9..bed8b9d57 100644 --- a/tests/test_pipeline_tpch_custom.py +++ b/tests/test_pipeline_tpch_custom.py @@ -4447,7 +4447,7 @@ ), id="explode_05", marks=pytest.mark.skip( - "Skipping until PyDough supports accessing subcollections from an EXPLODE operator." + "(gh #548) Skipping until PyDough supports accessing subcollections from an EXPLODE operator." ), ), pytest.param( @@ -4483,7 +4483,11 @@ "Customer", "000000005", ], - "nation_name": [""] * 10, + "nation_name": ["MOROCCO"] * 2 + + ["JORDAN"] * 2 + + ["ARGENTINA"] * 2 + + ["EGYPT"] * 2 + + ["CANADA"] * 2, } ), "explode_06", @@ -4492,9 +4496,137 @@ ), id="explode_06", marks=pytest.mark.skip( - "Skipping until PyDough supports accessing subcollections from an EXPLODE operator." + "(gh #548) Skipping until PyDough supports accessing subcollections from an EXPLODE operator." ), ), + pytest.param( + PyDoughPandasTest( + "result = (" + " customers" + " .CALCULATE(key)" + " .TOP_K(3, by=key.ASC())" + " .EXPLODE(comment, 'exp1', value_name='val1', index_name='idx1', version='string', delimiter='.')" + " .EXPLODE(val1, 'exp2', value_name='val2', index_name='idx2', version='string', delimiter=' ')" + " .EXPLODE(val2, 'exp3', value_name='val3', index_name='idx3', version='string', delimiter=',')" + " .WHERE(val3 != '')" + " .CALCULATE(idx1, idx2, idx3, val3)" + " .ORDER_BY(key.ASC(), idx1.ASC(), idx2.ASC(), idx3.ASC())" + ")", + "TPCH", + lambda: pd.DataFrame( + { + "key": [1] * 10 + [2] * 8 + [3] * 14, + "idx1": [ + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 2, + 2, + 2, + 1, + 1, + 2, + 2, + 2, + 2, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 2, + 2, + 3, + 3, + 3, + 3, + ], + "idx2": [ + 1, + 2, + 3, + 4, + 5, + 1, + 2, + 3, + 4, + 5, + 1, + 2, + 1, + 2, + 3, + 4, + 5, + 6, + 1, + 2, + 3, + 4, + 5, + 6, + 1, + 2, + 3, + 4, + 1, + 2, + 3, + 4, + ], + "idx3": [1] * 32, + "val3": [ + "to", + "the", + "even", + "regular", + "platelets", + "regular", + "ironic", + "epitaphs", + "nag", + "e", + "l", + "accounts", + "blithely", + "ironic", + "theodolites", + "integrate", + "boldly:", + "caref", + "deposits", + "eat", + "slyly", + "ironic", + "even", + "instructions", + "express", + "foxes", + "detect", + "slyly", + "blithely", + "even", + "accounts", + "abov", + ], + } + ), + "explode_07", + order_sensitive=True, + skipped_dialects={"ANSI", "SQLITE"}, + ), + id="explode_07", + ), pytest.param( PyDoughPandasTest( simple_range_1, diff --git a/tests/test_plan_refsols/explode_07.txt b/tests/test_plan_refsols/explode_07.txt new file mode 100644 index 000000000..ddc0890f1 --- /dev/null +++ b/tests/test_plan_refsols/explode_07.txt @@ -0,0 +1,7 @@ +ROOT(columns=[('idx1', idx1), ('idx2', idx2), ('idx3', idx3), ('val3', val3)], orderings=[(key):asc_first, (idx1):asc_first, (idx2):asc_first, (idx3):asc_first]) + EXPLODE(val2, value_name='val3', index_name='idx3', version='string', delimiter=',', filtering=True, is_distinct=False, columns={'idx1': idx1, 'idx2': idx2, 'idx3': idx3, 'key': key, 'val3': val3}) + EXPLODE(val1, value_name='val2', index_name='idx2', version='string', delimiter=' ', filtering=True, is_distinct=False, columns={'idx1': idx1, 'idx2': idx2, 'key': key, 'val2': val2}) + EXPLODE(comment, value_name='val1', index_name='idx1', version='string', delimiter='.', filtering=True, is_distinct=False, columns={'idx1': idx1, 'key': key, 'val1': val1}) + FILTER(condition=val3 != '':string, columns={'comment': comment, 'key': key}) + LIMIT(limit=3:numeric, columns={'comment': comment, 'key': key}, orderings=[(key):asc_first]) + SCAN(table=tpch.CUSTOMER, columns={'comment': c_comment, 'key': c_custkey}) diff --git a/tests/test_qualification.py b/tests/test_qualification.py index e855c5f25..3ff336008 100644 --- a/tests/test_qualification.py +++ b/tests/test_qualification.py @@ -1054,7 +1054,7 @@ """, id="good_explode_09", marks=pytest.mark.skip( - "Skipping until PyDough supports accessing subcollections from an EXPLODE operator." + "(gh #548) Skipping until PyDough supports accessing subcollections from an EXPLODE operator." ), ), ], From c727d6b76bfc6974f7203383df0a6dc246af54d7 Mon Sep 17 00:00:00 2001 From: knassre-bodo Date: Thu, 6 Aug 2026 10:53:22 -0700 Subject: [PATCH 21/44] Three explode tests working in Snowflake --- pydough/sqlglot/sqlglot_relational_visitor.py | 25 ++++++- .../base_transform_bindings.py | 31 +++++++++ .../databricks_transform_bindings.py | 5 ++ .../sf_transform_bindings.py | 65 +++++++++++++++++++ tests/test_pipeline_tpch_custom.py | 22 ++++--- tests/test_plan_refsols/explode_01.txt | 4 +- tests/test_plan_refsols/explode_04.txt | 16 +++-- .../test_sql_refsols/explode_01_snowflake.sql | 20 ++++++ .../test_sql_refsols/explode_03_snowflake.sql | 16 +++++ .../test_sql_refsols/explode_04_snowflake.sql | 44 +++++++++++++ 10 files changed, 232 insertions(+), 16 deletions(-) create mode 100644 tests/test_sql_refsols/explode_01_snowflake.sql create mode 100644 tests/test_sql_refsols/explode_03_snowflake.sql create mode 100644 tests/test_sql_refsols/explode_04_snowflake.sql diff --git a/pydough/sqlglot/sqlglot_relational_visitor.py b/pydough/sqlglot/sqlglot_relational_visitor.py index b4846e3ec..0d28c6cc9 100644 --- a/pydough/sqlglot/sqlglot_relational_visitor.py +++ b/pydough/sqlglot/sqlglot_relational_visitor.py @@ -583,7 +583,30 @@ def visit_generated_table(self, generated_table: "GeneratedTable") -> None: self._stack.append(query) def visit_explode(self, explode: Explode) -> None: - raise NotImplementedError() + self.visit_inputs(explode) + input_expr: Select = self._stack.pop() + explode_expr: SQLGlotExpression = self._expr_visitor.relational_to_sqlglot( + explode.explode_data + ) + exprs: list[SQLGlotExpression] = [ + self._expr_visitor.relational_to_sqlglot(col, alias) + for alias, col in sorted(explode.columns.items()) + ] + val_index: int | None = None + idx_index: int | None = None + for i, (_, expr) in enumerate(sorted(explode.columns.items())): + if isinstance(expr, ColumnReference): + if expr.name == explode.explode_spec.value_name: + val_index = i + elif ( + explode.explode_spec.index_name is not None + and expr.name == explode.explode_spec.index_name + ): + idx_index = i + query: SQLGlotExpression = self._expr_visitor._bindings.convert_explode( + input_expr, explode_expr, explode.explode_spec, exprs, val_index, idx_index + ) + self._stack.append(query) def relational_to_sqlglot(self, root: RelationalRoot) -> SQLGlotExpression: """ diff --git a/pydough/sqlglot/transform_bindings/base_transform_bindings.py b/pydough/sqlglot/transform_bindings/base_transform_bindings.py index 005f5f648..d0eeb1d6b 100644 --- a/pydough/sqlglot/transform_bindings/base_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/base_transform_bindings.py @@ -35,6 +35,7 @@ from pydough.user_collections.range_collection import RangeGeneratedCollection from pydough.user_collections.user_collections import PyDoughUserGeneratedCollection from pydough.user_collections.view_collection import ViewGeneratedCollection +from pydough.utilities import ExplodeSpec from .sqlglot_transform_utils import ( DateTimeUnit, @@ -2334,6 +2335,36 @@ def create_empty_singleton(self) -> SQLGlotExpression: .from_(sqlglot_expressions.values([sqlglot_expressions.convert((None,))])) ) + def convert_explode( + self, + input_expr: SQLGlotExpression, + explode_expr: SQLGlotExpression, + explode_spec: ExplodeSpec, + exprs: list[SQLGlotExpression], + val_index: int | None, + idx_index: int | None, + ) -> SQLGlotExpression: + """ + Converts a PyDough EXPLODE operation call to a SQLGlot expression that + unravels an array or string into multiple rows (i.e. a lateral join). + + Args: + `input_expr`: The subquery containing the data being exploded. + `explode_expr`: The expression to explode (e.g., an array or string + column). + `explode_spec`: The specification of how to explode the data. + `exprs`: The list of SQLGlot expressions representing the columns of + the data besides the exploded output. + `val_index`: The index within `exprs` that should be used to store + the exploded data in the output. If None, not included. + `idx_index`: The index within `exprs` that should be used to store + the exploded data's index in the output. If None, not included. + + Returns: + A SQLGlotExpression representing the exploded data. + """ + raise PyDoughSQLException("EXPLODE is not supported in this dialect") + def convert_user_generated_collection( self, collection: PyDoughUserGeneratedCollection, diff --git a/pydough/sqlglot/transform_bindings/databricks_transform_bindings.py b/pydough/sqlglot/transform_bindings/databricks_transform_bindings.py index 50ba9eedf..bcf8fec61 100644 --- a/pydough/sqlglot/transform_bindings/databricks_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/databricks_transform_bindings.py @@ -113,6 +113,11 @@ def convert_integer( to=sqlglot_expressions.DataType.build("BIGINT"), ) + def convert_listof( + self, args: SQLGlotExpression, types: list[PyDoughType] + ) -> SQLGlotExpression: + return sqlglot_expressions.ArrayAgg(this=args[0]) + def generate_dataframe_item_dialect_expression( self, item: Any, item_type: PyDoughType ) -> SQLGlotExpression: diff --git a/pydough/sqlglot/transform_bindings/sf_transform_bindings.py b/pydough/sqlglot/transform_bindings/sf_transform_bindings.py index 3deb26f35..43d7d262b 100644 --- a/pydough/sqlglot/transform_bindings/sf_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/sf_transform_bindings.py @@ -9,6 +9,14 @@ from typing import Any import sqlglot.expressions as sqlglot_expressions +from sqlglot.expressions import ( + Anonymous, + Identifier, + Lateral, + Select, + Subquery, + TableAlias, +) from sqlglot.expressions import Expression as SQLGlotExpression import pydough.pydough_operators as pydop @@ -21,6 +29,7 @@ from pydough.types.datetime_type import DatetimeType from pydough.types.numeric_type import NumericType from pydough.user_collections.range_collection import RangeGeneratedCollection +from pydough.utilities import ExplodeSpec from .base_transform_bindings import BaseTransformBindings from .sqlglot_transform_utils import ( @@ -258,6 +267,62 @@ def convert_datediff( # For other units, use base implementation return super().convert_datediff(args, types) + def convert_explode( + self, + input_expr: SQLGlotExpression, + explode_expr: SQLGlotExpression, + explode_spec: ExplodeSpec, + exprs: list[SQLGlotExpression], + val_index: int | None, + idx_index: int | None, + ) -> SQLGlotExpression: + column_exprs: list[SQLGlotExpression] = [*exprs] + lateral_prefix: str = "_L" + if val_index is not None: + column_exprs[val_index] = sqlglot_expressions.Alias( + this=sqlglot_expressions.Column( + this=sqlglot_expressions.Identifier(this="VALUE"), + table=sqlglot_expressions.Identifier(this=lateral_prefix), + ), + alias=sqlglot_expressions.Identifier(this=explode_spec.value_name), + ) + if idx_index is not None and explode_spec.index_name is not None: + column_exprs[idx_index] = sqlglot_expressions.Alias( + this=sqlglot_expressions.Column( + this=sqlglot_expressions.Identifier(this="INDEX"), + table=sqlglot_expressions.Identifier(this=lateral_prefix), + ), + alias=sqlglot_expressions.Identifier(this=explode_spec.index_name), + ) + + explode_op: SQLGlotExpression + if explode_spec.version == "array": + explode_op = Anonymous(this="FLATTEN", expressions=[explode_expr]) + else: + assert ( + explode_spec.version == "string" and explode_spec.delimiter is not None + ) + explode_op = Anonymous( + this="SPLIT_TO_TABLE", + expressions=[ + explode_expr, + sqlglot_expressions.Literal.string(explode_spec.delimiter), + ], + ) + result = ( + Select() + .select(*column_exprs) + .from_(Subquery(this=input_expr)) + .join( + Lateral( + this=explode_op, + alias=TableAlias(this=Identifier(this=lateral_prefix)), + ) + ) + ) + + return result + def convert_user_generated_range( self, collection: RangeGeneratedCollection ) -> SQLGlotExpression: diff --git a/tests/test_pipeline_tpch_custom.py b/tests/test_pipeline_tpch_custom.py index bed8b9d57..536b3606f 100644 --- a/tests/test_pipeline_tpch_custom.py +++ b/tests/test_pipeline_tpch_custom.py @@ -4243,6 +4243,7 @@ "result = (" " array_data" " .EXPLODE(nation_names, 'exploded_nations', index_name='nation_idx', value_name='nation_name', version='array', filtering=False, is_distinct=True)" + " .CALCULATE(region_name, nation_names, nation_idx, nation_name)" " .ORDER_BY(region_name, nation_idx)" ")", "TPCH", @@ -4253,7 +4254,7 @@ + ["ASIA"] * 5 + ["EUROPE"] * 5 + ["MIDDLE EAST"] * 5, - "nation_names": [ + "nation_names": ( [["ALGERIA", "ETHIOPIA", "KENYA", "MOROCCO", "MOZAMBIQUE"]] * 5 + [ @@ -4277,15 +4278,14 @@ ] ] * 5 - + [["EGYPT", "IRAN", "IRAQ", "JORDAN", "SAUDI ARABIA"]] * 5, - ], + + [["EGYPT", "IRAN", "IRAQ", "JORDAN", "SAUDI ARABIA"]] * 5 + ), "nation_idx": list(range(5)) * 5, "nation_name": [ "ALGERIA", "ETHIOPIA", "KENYA", "MOROCCO", - "MOROCCO", "MOZAMBIQUE", "ARGENTINA", "BRAZIL", @@ -4345,7 +4345,7 @@ ), "explode_02", order_sensitive=True, - skipped_dialects={"ANSI", "SQLITE"}, + skipped_dialects={"ANSI", "SQLITE", "SNOWFLAKE"}, kwargs={ "array_df": pd.DataFrame( { @@ -4369,7 +4369,6 @@ "TPCH", lambda: pd.DataFrame( { - "idx": [1, 2] * 5, "val": [ "Customer", "000000001", @@ -4382,6 +4381,7 @@ "Customer", "000000005", ], + "idx": [1, 2] * 5, } ), "explode_03", @@ -4392,10 +4392,12 @@ ), pytest.param( PyDoughPandasTest( - "exploded_data = EXPLODE(name, 'exploded_names', index_name='idx', value_name='val', version='string', delimiter='I')\n" + "exploded_i = EXPLODE(name, 'exploded_names', index_name='idx', value_name='val', version='string', delimiter='I')\n" + "exploded_e = EXPLODE(name, 'exploded_names', index_name='idx', value_name='val', version='string', delimiter='E')\n" + "exploded_space = EXPLODE(name, 'exploded_names', index_name='idx', value_name='val', version='string', delimiter=' ')\n" "result = (" " regions" - " .CALCULATE(region_name=name, n_chunks=COUNT(exploded_data))" + " .CALCULATE(region_name=name, n_e_chunks=COUNT(exploded_e), n_i_chunks=COUNT(exploded_i), n_space_chunks=COUNT(exploded_space))" " .ORDER_BY(region_name)" ")", "TPCH", @@ -4408,7 +4410,9 @@ "EUROPE", "MIDDLE EAST", ], - "n_chunks": [2, 2, 2, 1, 1], + "n_e_chunks": [1, 2, 1, 3, 3], + "n_i_chunks": [2, 2, 2, 1, 2], + "n_space_chunks": [1, 1, 1, 1, 2], } ), "explode_04", diff --git a/tests/test_plan_refsols/explode_01.txt b/tests/test_plan_refsols/explode_01.txt index 45f9c6095..519945caa 100644 --- a/tests/test_plan_refsols/explode_01.txt +++ b/tests/test_plan_refsols/explode_01.txt @@ -1,5 +1,5 @@ -ROOT(columns=[('nation_name', nation_name), ('nation_idx', nation_idx)], orderings=[(region_name):asc_first, (nation_idx):asc_first]) - EXPLODE(nation_names, value_name='nation_name', index_name='nation_idx', version='array', filtering=False, is_distinct=True, columns={'nation_idx': nation_idx, 'nation_name': nation_name, 'region_name': region_name}) +ROOT(columns=[('region_name', region_name), ('nation_names', nation_names), ('nation_idx', nation_idx), ('nation_name', nation_name)], orderings=[(region_name):asc_first, (nation_idx):asc_first]) + EXPLODE(nation_names, value_name='nation_name', index_name='nation_idx', version='array', filtering=False, is_distinct=True, columns={'nation_idx': nation_idx, 'nation_name': nation_name, 'nation_names': nation_names, 'region_name': region_name}) JOIN(condition=t0.key == t1.region_key, type=INNER, cardinality=SINGULAR_ACCESS, reverse_cardinality=SINGULAR_ACCESS, columns={'nation_names': t1.nation_names, 'region_name': t0.region_name}) SCAN(table=tpch.REGION, columns={'key': r_regionkey, 'region_name': r_name}) AGGREGATE(keys={'region_key': region_key}, aggregations={'nation_names': LISTOF(name)}) diff --git a/tests/test_plan_refsols/explode_04.txt b/tests/test_plan_refsols/explode_04.txt index 7ee25fa1f..4ddc475fa 100644 --- a/tests/test_plan_refsols/explode_04.txt +++ b/tests/test_plan_refsols/explode_04.txt @@ -1,6 +1,14 @@ -ROOT(columns=[('region_name', r_name), ('n_chunks', DEFAULT_TO(n_rows, 0:numeric))], orderings=[(r_name):asc_first]) - JOIN(condition=t0.r_regionkey == t1.key, type=LEFT, cardinality=SINGULAR_FILTER, reverse_cardinality=SINGULAR_ACCESS, columns={'n_rows': t1.n_rows, 'r_name': t0.r_name}) - SCAN(table=tpch.REGION, columns={'r_name': r_name, 'r_regionkey': r_regionkey}) +ROOT(columns=[('region_name', r_name), ('n_e_chunks', DEFAULT_TO(n_rows, 0:numeric)), ('n_i_chunks', DEFAULT_TO(agg_1, 0:numeric)), ('n_space_chunks', DEFAULT_TO(agg_2, 0:numeric))], orderings=[(r_name):asc_first]) + JOIN(condition=t0.r_regionkey == t1.key, type=LEFT, cardinality=SINGULAR_FILTER, reverse_cardinality=SINGULAR_ACCESS, columns={'agg_1': t0.agg_1, 'agg_2': t1.n_rows, 'n_rows': t0.n_rows, 'r_name': t0.r_name}) + JOIN(condition=t0.r_regionkey == t1.key, type=LEFT, cardinality=SINGULAR_FILTER, reverse_cardinality=SINGULAR_ACCESS, columns={'agg_1': t1.n_rows, 'n_rows': t0.n_rows, 'r_name': t0.r_name, 'r_regionkey': t0.r_regionkey}) + JOIN(condition=t0.r_regionkey == t1.key, type=LEFT, cardinality=SINGULAR_FILTER, reverse_cardinality=SINGULAR_ACCESS, columns={'n_rows': t1.n_rows, 'r_name': t0.r_name, 'r_regionkey': t0.r_regionkey}) + SCAN(table=tpch.REGION, columns={'r_name': r_name, 'r_regionkey': r_regionkey}) + AGGREGATE(keys={'key': key}, aggregations={'n_rows': COUNT()}) + EXPLODE(name, value_name='val', index_name='idx', version='string', delimiter='E', filtering=True, is_distinct=False, columns={'idx': idx, 'key': key, 'val': val}) + SCAN(table=tpch.REGION, columns={'key': r_regionkey, 'name': r_name}) + AGGREGATE(keys={'key': key}, aggregations={'n_rows': COUNT()}) + EXPLODE(name, value_name='val', index_name='idx', version='string', delimiter='I', filtering=True, is_distinct=False, columns={'idx': idx, 'key': key, 'val': val}) + SCAN(table=tpch.REGION, columns={'key': r_regionkey, 'name': r_name}) AGGREGATE(keys={'key': key}, aggregations={'n_rows': COUNT()}) - EXPLODE(name, value_name='val', index_name='idx', version='string', delimiter='I', filtering=True, is_distinct=False, columns={'idx': idx, 'key': key, 'val': val}) + EXPLODE(name, value_name='val', index_name='idx', version='string', delimiter=' ', filtering=True, is_distinct=False, columns={'idx': idx, 'key': key, 'val': val}) SCAN(table=tpch.REGION, columns={'key': r_regionkey, 'name': r_name}) diff --git a/tests/test_sql_refsols/explode_01_snowflake.sql b/tests/test_sql_refsols/explode_01_snowflake.sql new file mode 100644 index 000000000..aa547d6c7 --- /dev/null +++ b/tests/test_sql_refsols/explode_01_snowflake.sql @@ -0,0 +1,20 @@ +WITH _s1 AS ( + SELECT + n_regionkey AS region_key, + ARRAY_AGG(n_name) AS nation_names + FROM tpch.nation + GROUP BY + 1 +) +SELECT + region.r_name AS region_name, + _s1.nation_names, + _l.index AS nation_idx, + _l.value AS nation_name +FROM tpch.region AS region +JOIN _s1 AS _s1 + ON _s1.region_key = region.r_regionkey +CROSS JOIN LATERAL FLATTEN(_s1.nation_names) AS _l(seq, key, path, index, value, this) +ORDER BY + 1 NULLS FIRST, + 3 NULLS FIRST diff --git a/tests/test_sql_refsols/explode_03_snowflake.sql b/tests/test_sql_refsols/explode_03_snowflake.sql new file mode 100644 index 000000000..3554db688 --- /dev/null +++ b/tests/test_sql_refsols/explode_03_snowflake.sql @@ -0,0 +1,16 @@ +WITH _q_0 AS ( + SELECT + c_name AS name + FROM tpch.customer + ORDER BY + c_custkey NULLS FIRST + LIMIT 5 +) +SELECT + _l.value AS val, + _l.index AS idx +FROM _q_0 AS _q_0 +CROSS JOIN LATERAL SPLIT_TO_TABLE(_q_0.name, '#') AS _l +ORDER BY + _q_0.name NULLS FIRST, + 2 NULLS FIRST diff --git a/tests/test_sql_refsols/explode_04_snowflake.sql b/tests/test_sql_refsols/explode_04_snowflake.sql new file mode 100644 index 000000000..79267f1d8 --- /dev/null +++ b/tests/test_sql_refsols/explode_04_snowflake.sql @@ -0,0 +1,44 @@ +WITH _q_0 AS ( + SELECT + r_regionkey AS key, + r_name AS name + FROM tpch.region +), _s1 AS ( + SELECT + _q_0.key, + COUNT(*) AS n_rows + FROM _q_0 AS _q_0 + CROSS JOIN LATERAL SPLIT_TO_TABLE(_q_0.name, 'E') AS _l + GROUP BY + 1 +), _s3 AS ( + SELECT + _q_1.key, + COUNT(*) AS n_rows + FROM _q_0 AS _q_1 + CROSS JOIN LATERAL SPLIT_TO_TABLE(_q_1.name, 'I') AS _l + GROUP BY + 1 +), _s5 AS ( + SELECT + _q_2.key, + COUNT(*) AS n_rows + FROM _q_0 AS _q_2 + CROSS JOIN LATERAL SPLIT_TO_TABLE(_q_2.name, ' ') AS _l + GROUP BY + 1 +) +SELECT + region.r_name AS region_name, + COALESCE(_s1.n_rows, 0) AS n_e_chunks, + COALESCE(_s3.n_rows, 0) AS n_i_chunks, + COALESCE(_s5.n_rows, 0) AS n_space_chunks +FROM tpch.region AS region +LEFT JOIN _s1 AS _s1 + ON _s1.key = region.r_regionkey +LEFT JOIN _s3 AS _s3 + ON _s3.key = region.r_regionkey +LEFT JOIN _s5 AS _s5 + ON _s5.key = region.r_regionkey +ORDER BY + 1 NULLS FIRST From ac53caf2bd6d9060fee066d4377da6a9ca686a4e Mon Sep 17 00:00:00 2001 From: knassre-bodo Date: Thu, 6 Aug 2026 11:46:40 -0700 Subject: [PATCH 22/44] Adding POSTGRES support --- .../postgres_transform_bindings.py | 75 +++++++++++++++++++ .../sf_transform_bindings.py | 14 +++- tests/test_pipeline_tpch_custom.py | 4 +- .../test_sql_refsols/explode_01_postgres.sql | 20 +++++ .../test_sql_refsols/explode_02_postgres.sql | 16 ++++ .../test_sql_refsols/explode_03_postgres.sql | 16 ++++ .../test_sql_refsols/explode_03_snowflake.sql | 2 +- .../test_sql_refsols/explode_04_postgres.sql | 44 +++++++++++ 8 files changed, 184 insertions(+), 7 deletions(-) create mode 100644 tests/test_sql_refsols/explode_01_postgres.sql create mode 100644 tests/test_sql_refsols/explode_02_postgres.sql create mode 100644 tests/test_sql_refsols/explode_03_postgres.sql create mode 100644 tests/test_sql_refsols/explode_04_postgres.sql diff --git a/pydough/sqlglot/transform_bindings/postgres_transform_bindings.py b/pydough/sqlglot/transform_bindings/postgres_transform_bindings.py index 54b846452..10f5df167 100644 --- a/pydough/sqlglot/transform_bindings/postgres_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/postgres_transform_bindings.py @@ -10,6 +10,14 @@ import sqlglot.expressions as sqlglot_expressions from sqlglot import parse_one from sqlglot.expressions import Expression as SQLGlotExpression +from sqlglot.expressions import ( + Identifier, + Lateral, + Select, + Subquery, + TableAlias, + Unnest, +) import pydough.pydough_operators as pydop from pydough.relational.relational_expressions.literal_expression import ( @@ -21,6 +29,7 @@ from pydough.types.datetime_type import DatetimeType from pydough.types.numeric_type import NumericType from pydough.user_collections.range_collection import RangeGeneratedCollection +from pydough.utilities import ExplodeSpec from .base_transform_bindings import BaseTransformBindings from .sqlglot_transform_utils import ( @@ -67,6 +76,72 @@ def convert_call_to_sqlglot( return super().convert_call_to_sqlglot(operator, args, types) + def convert_explode( + self, + input_expr: SQLGlotExpression, + explode_expr: SQLGlotExpression, + explode_spec: ExplodeSpec, + exprs: list[SQLGlotExpression], + val_index: int | None, + idx_index: int | None, + ) -> SQLGlotExpression: + column_exprs: list[SQLGlotExpression] = [*exprs] + lateral_prefix: str = "_L" + if val_index is not None: + column_exprs[val_index] = sqlglot_expressions.Alias( + this=sqlglot_expressions.Column( + this=sqlglot_expressions.Identifier(this="val"), + table=sqlglot_expressions.Identifier(this=lateral_prefix), + ), + alias=sqlglot_expressions.Identifier(this=explode_spec.value_name), + ) + if idx_index is not None and explode_spec.index_name is not None: + column_exprs[idx_index] = sqlglot_expressions.Alias( + this=sqlglot_expressions.Sub( + this=sqlglot_expressions.Column( + this=sqlglot_expressions.Identifier(this="idx"), + table=sqlglot_expressions.Identifier(this=lateral_prefix), + ), + expression=sqlglot_expressions.Literal.number(1), + ), + alias=sqlglot_expressions.Identifier(this=explode_spec.index_name), + ) + + if explode_spec.version == "string": + assert explode_spec.delimiter is not None, ( + "Delimiter must be provided for string explode." + ) + explode_expr = sqlglot_expressions.Anonymous( + this="STRING_TO_ARRAY", + expressions=[ + explode_expr, + sqlglot_expressions.Literal.string(explode_spec.delimiter), + ], + ) + explode_op: SQLGlotExpression = Unnest( + expressions=[explode_expr], + offset=sqlglot_expressions.Literal.number(1), + alias=TableAlias( + this=Identifier(this=lateral_prefix), + columns=[ + sqlglot_expressions.Identifier(this="val"), + sqlglot_expressions.Identifier(this="idx"), + ], + ), + ) + result = ( + Select() + .select(*column_exprs) + .from_(Subquery(this=input_expr)) + .join( + Lateral( + this=explode_op, + ) + ) + ) + + return result + def convert_listof( self, args: SQLGlotExpression, types: list[PyDoughType] ) -> SQLGlotExpression: diff --git a/pydough/sqlglot/transform_bindings/sf_transform_bindings.py b/pydough/sqlglot/transform_bindings/sf_transform_bindings.py index 43d7d262b..3f793904f 100644 --- a/pydough/sqlglot/transform_bindings/sf_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/sf_transform_bindings.py @@ -287,11 +287,17 @@ def convert_explode( alias=sqlglot_expressions.Identifier(this=explode_spec.value_name), ) if idx_index is not None and explode_spec.index_name is not None: + idx_expr: SQLGlotExpression = sqlglot_expressions.Column( + this=sqlglot_expressions.Identifier(this="INDEX"), + table=sqlglot_expressions.Identifier(this=lateral_prefix), + ) + if explode_spec.version == "string": + idx_expr = sqlglot_expressions.Sub( + this=idx_expr, + expression=sqlglot_expressions.Literal.number(1), + ) column_exprs[idx_index] = sqlglot_expressions.Alias( - this=sqlglot_expressions.Column( - this=sqlglot_expressions.Identifier(this="INDEX"), - table=sqlglot_expressions.Identifier(this=lateral_prefix), - ), + this=idx_expr, alias=sqlglot_expressions.Identifier(this=explode_spec.index_name), ) diff --git a/tests/test_pipeline_tpch_custom.py b/tests/test_pipeline_tpch_custom.py index 536b3606f..26e0d2eab 100644 --- a/tests/test_pipeline_tpch_custom.py +++ b/tests/test_pipeline_tpch_custom.py @@ -4339,7 +4339,7 @@ [5, 6], [5, 6], ], - "arr_idx": [1, 1, 2, 3, 4, 1, 2], + "arr_idx": [0, 0, 1, 2, 3, 0, 1], "arr_val": [1, 2, 3, None, 4, 5, 6], } ), @@ -4381,7 +4381,7 @@ "Customer", "000000005", ], - "idx": [1, 2] * 5, + "idx": [0, 1] * 5, } ), "explode_03", diff --git a/tests/test_sql_refsols/explode_01_postgres.sql b/tests/test_sql_refsols/explode_01_postgres.sql new file mode 100644 index 000000000..853bb781a --- /dev/null +++ b/tests/test_sql_refsols/explode_01_postgres.sql @@ -0,0 +1,20 @@ +WITH _s1 AS ( + SELECT + ARRAY_AGG(n_name) AS nation_names, + n_regionkey AS region_key + FROM tpch.nation + GROUP BY + 2 +) +SELECT + region.r_name AS region_name, + _s1.nation_names, + _l.idx - 1 AS nation_idx, + _l.val AS nation_name +FROM tpch.region AS region +JOIN _s1 AS _s1 + ON _s1.region_key = region.r_regionkey +CROSS JOIN LATERAL UNNEST(_s1.nation_names) WITH ORDINALITY AS _l(val, idx) +ORDER BY + 1 NULLS FIRST, + 3 NULLS FIRST diff --git a/tests/test_sql_refsols/explode_02_postgres.sql b/tests/test_sql_refsols/explode_02_postgres.sql new file mode 100644 index 000000000..f733a5219 --- /dev/null +++ b/tests/test_sql_refsols/explode_02_postgres.sql @@ -0,0 +1,16 @@ +SELECT + tbl.key, + tbl.arr, + _l.idx - 1 AS arr_idx, + _l.val AS arr_val +FROM (VALUES + ('A', ARRAY[1]), + ('B', ( + ARRAY[0] + )[1 : 0]), + ('C', ARRAY[2, 3, NULL, 4]), + ('D', ARRAY[5, 6])) AS tbl(key, arr) +CROSS JOIN LATERAL UNNEST(tbl.arr) WITH ORDINALITY AS _l(val, idx) +ORDER BY + 1 NULLS FIRST, + 3 NULLS FIRST diff --git a/tests/test_sql_refsols/explode_03_postgres.sql b/tests/test_sql_refsols/explode_03_postgres.sql new file mode 100644 index 000000000..c2c1cac48 --- /dev/null +++ b/tests/test_sql_refsols/explode_03_postgres.sql @@ -0,0 +1,16 @@ +WITH _q_0 AS ( + SELECT + c_name AS name + FROM tpch.customer + ORDER BY + c_custkey NULLS FIRST + LIMIT 5 +) +SELECT + _l.val, + _l.idx - 1 AS idx +FROM _q_0 AS _q_0 +CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_q_0.name, '#')) WITH ORDINALITY AS _l(val, idx) +ORDER BY + _q_0.name NULLS FIRST, + 2 NULLS FIRST diff --git a/tests/test_sql_refsols/explode_03_snowflake.sql b/tests/test_sql_refsols/explode_03_snowflake.sql index 3554db688..67a868dc1 100644 --- a/tests/test_sql_refsols/explode_03_snowflake.sql +++ b/tests/test_sql_refsols/explode_03_snowflake.sql @@ -8,7 +8,7 @@ WITH _q_0 AS ( ) SELECT _l.value AS val, - _l.index AS idx + _l.index - 1 AS idx FROM _q_0 AS _q_0 CROSS JOIN LATERAL SPLIT_TO_TABLE(_q_0.name, '#') AS _l ORDER BY diff --git a/tests/test_sql_refsols/explode_04_postgres.sql b/tests/test_sql_refsols/explode_04_postgres.sql new file mode 100644 index 000000000..c93c07450 --- /dev/null +++ b/tests/test_sql_refsols/explode_04_postgres.sql @@ -0,0 +1,44 @@ +WITH _q_0 AS ( + SELECT + r_regionkey AS key, + r_name AS name + FROM tpch.region +), _s1 AS ( + SELECT + _q_0.key, + COUNT(*) AS n_rows + FROM _q_0 AS _q_0 + CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_q_0.name, 'E')) WITH ORDINALITY AS _l(val, idx) + GROUP BY + 1 +), _s3 AS ( + SELECT + _q_1.key, + COUNT(*) AS n_rows + FROM _q_0 AS _q_1 + CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_q_1.name, 'I')) WITH ORDINALITY AS _l(val, idx) + GROUP BY + 1 +), _s5 AS ( + SELECT + _q_2.key, + COUNT(*) AS n_rows + FROM _q_0 AS _q_2 + CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_q_2.name, ' ')) WITH ORDINALITY AS _l(val, idx) + GROUP BY + 1 +) +SELECT + region.r_name AS region_name, + COALESCE(_s1.n_rows, 0) AS n_e_chunks, + COALESCE(_s3.n_rows, 0) AS n_i_chunks, + COALESCE(_s5.n_rows, 0) AS n_space_chunks +FROM tpch.region AS region +LEFT JOIN _s1 AS _s1 + ON _s1.key = region.r_regionkey +LEFT JOIN _s3 AS _s3 + ON _s3.key = region.r_regionkey +LEFT JOIN _s5 AS _s5 + ON _s5.key = region.r_regionkey +ORDER BY + 1 NULLS FIRST From 799534d907b60b96cf79d12aeec16a36e7356f6c Mon Sep 17 00:00:00 2001 From: knassre-bodo Date: Fri, 7 Aug 2026 10:26:54 -0700 Subject: [PATCH 23/44] Added generated alias, fixed some of problems with multi-explode --- pydough/sqlglot/sqlglot_relational_visitor.py | 8 +- .../base_transform_bindings.py | 3 + .../postgres_transform_bindings.py | 8 +- .../sf_transform_bindings.py | 8 +- tests/test_pipeline_tpch_custom.py | 79 ++++++++++++++----- tests/test_plan_refsols/explode_07.txt | 12 ++- tests/test_plan_refsols/explode_08.txt | 7 ++ .../test_sql_refsols/explode_01_postgres.sql | 6 +- .../test_sql_refsols/explode_01_snowflake.sql | 6 +- .../test_sql_refsols/explode_02_postgres.sql | 6 +- .../test_sql_refsols/explode_03_postgres.sql | 6 +- .../test_sql_refsols/explode_03_snowflake.sql | 6 +- .../test_sql_refsols/explode_04_postgres.sql | 26 +++--- .../test_sql_refsols/explode_04_snowflake.sql | 26 +++--- .../test_sql_refsols/explode_07_postgres.sql | 21 +++++ .../test_sql_refsols/explode_07_snowflake.sql | 21 +++++ .../test_sql_refsols/explode_08_postgres.sql | 26 ++++++ .../test_sql_refsols/explode_08_snowflake.sql | 26 ++++++ 18 files changed, 223 insertions(+), 78 deletions(-) create mode 100644 tests/test_plan_refsols/explode_08.txt create mode 100644 tests/test_sql_refsols/explode_07_postgres.sql create mode 100644 tests/test_sql_refsols/explode_07_snowflake.sql create mode 100644 tests/test_sql_refsols/explode_08_postgres.sql create mode 100644 tests/test_sql_refsols/explode_08_snowflake.sql diff --git a/pydough/sqlglot/sqlglot_relational_visitor.py b/pydough/sqlglot/sqlglot_relational_visitor.py index 0d28c6cc9..fab7e4419 100644 --- a/pydough/sqlglot/sqlglot_relational_visitor.py +++ b/pydough/sqlglot/sqlglot_relational_visitor.py @@ -604,7 +604,13 @@ def visit_explode(self, explode: Explode) -> None: ): idx_index = i query: SQLGlotExpression = self._expr_visitor._bindings.convert_explode( - input_expr, explode_expr, explode.explode_spec, exprs, val_index, idx_index + input_expr, + explode_expr, + explode.explode_spec, + exprs, + val_index, + idx_index, + self._generate_table_alias(), ) self._stack.append(query) diff --git a/pydough/sqlglot/transform_bindings/base_transform_bindings.py b/pydough/sqlglot/transform_bindings/base_transform_bindings.py index d0eeb1d6b..1ddefac27 100644 --- a/pydough/sqlglot/transform_bindings/base_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/base_transform_bindings.py @@ -2343,6 +2343,7 @@ def convert_explode( exprs: list[SQLGlotExpression], val_index: int | None, idx_index: int | None, + lateral_alias: str, ) -> SQLGlotExpression: """ Converts a PyDough EXPLODE operation call to a SQLGlot expression that @@ -2359,6 +2360,8 @@ def convert_explode( the exploded data in the output. If None, not included. `idx_index`: The index within `exprs` that should be used to store the exploded data's index in the output. If None, not included. + `lateral_alias`: A name given to a LATERAL operator, if needed, to + avoid name collisions. Returns: A SQLGlotExpression representing the exploded data. diff --git a/pydough/sqlglot/transform_bindings/postgres_transform_bindings.py b/pydough/sqlglot/transform_bindings/postgres_transform_bindings.py index 10f5df167..c90829f71 100644 --- a/pydough/sqlglot/transform_bindings/postgres_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/postgres_transform_bindings.py @@ -84,14 +84,14 @@ def convert_explode( exprs: list[SQLGlotExpression], val_index: int | None, idx_index: int | None, + lateral_alias: str, ) -> SQLGlotExpression: column_exprs: list[SQLGlotExpression] = [*exprs] - lateral_prefix: str = "_L" if val_index is not None: column_exprs[val_index] = sqlglot_expressions.Alias( this=sqlglot_expressions.Column( this=sqlglot_expressions.Identifier(this="val"), - table=sqlglot_expressions.Identifier(this=lateral_prefix), + table=sqlglot_expressions.Identifier(this=lateral_alias), ), alias=sqlglot_expressions.Identifier(this=explode_spec.value_name), ) @@ -100,7 +100,7 @@ def convert_explode( this=sqlglot_expressions.Sub( this=sqlglot_expressions.Column( this=sqlglot_expressions.Identifier(this="idx"), - table=sqlglot_expressions.Identifier(this=lateral_prefix), + table=sqlglot_expressions.Identifier(this=lateral_alias), ), expression=sqlglot_expressions.Literal.number(1), ), @@ -122,7 +122,7 @@ def convert_explode( expressions=[explode_expr], offset=sqlglot_expressions.Literal.number(1), alias=TableAlias( - this=Identifier(this=lateral_prefix), + this=Identifier(this=lateral_alias), columns=[ sqlglot_expressions.Identifier(this="val"), sqlglot_expressions.Identifier(this="idx"), diff --git a/pydough/sqlglot/transform_bindings/sf_transform_bindings.py b/pydough/sqlglot/transform_bindings/sf_transform_bindings.py index 3f793904f..f9ba2de2b 100644 --- a/pydough/sqlglot/transform_bindings/sf_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/sf_transform_bindings.py @@ -275,21 +275,21 @@ def convert_explode( exprs: list[SQLGlotExpression], val_index: int | None, idx_index: int | None, + lateral_alias: str, ) -> SQLGlotExpression: column_exprs: list[SQLGlotExpression] = [*exprs] - lateral_prefix: str = "_L" if val_index is not None: column_exprs[val_index] = sqlglot_expressions.Alias( this=sqlglot_expressions.Column( this=sqlglot_expressions.Identifier(this="VALUE"), - table=sqlglot_expressions.Identifier(this=lateral_prefix), + table=sqlglot_expressions.Identifier(this=lateral_alias), ), alias=sqlglot_expressions.Identifier(this=explode_spec.value_name), ) if idx_index is not None and explode_spec.index_name is not None: idx_expr: SQLGlotExpression = sqlglot_expressions.Column( this=sqlglot_expressions.Identifier(this="INDEX"), - table=sqlglot_expressions.Identifier(this=lateral_prefix), + table=sqlglot_expressions.Identifier(this=lateral_alias), ) if explode_spec.version == "string": idx_expr = sqlglot_expressions.Sub( @@ -322,7 +322,7 @@ def convert_explode( .join( Lateral( this=explode_op, - alias=TableAlias(this=Identifier(this=lateral_prefix)), + alias=TableAlias(this=Identifier(this=lateral_alias)), ) ) ) diff --git a/tests/test_pipeline_tpch_custom.py b/tests/test_pipeline_tpch_custom.py index 26e0d2eab..5e76212a3 100644 --- a/tests/test_pipeline_tpch_custom.py +++ b/tests/test_pipeline_tpch_custom.py @@ -4503,6 +4503,43 @@ "(gh #548) Skipping until PyDough supports accessing subcollections from an EXPLODE operator." ), ), + pytest.param( + PyDoughPandasTest( + "result = (" + " customers" + " .CALCULATE(key)" + " .TOP_K(3, by=key.ASC())" + " .EXPLODE(comment, 'exp1', value_name='val1', index_name='idx1', version='string', delimiter='.')" + " .EXPLODE(val1, 'exp2', value_name='val2', index_name='idx2', version='string', delimiter=',')" + " .CALCULATE(key, idx1, idx2, val2)" + " .ORDER_BY(key.ASC(), idx1.ASC(), idx2.ASC())" + ")", + "TPCH", + lambda: pd.DataFrame( + { + "key": [1, 1, 1, 1, 2, 2, 3, 3, 3, 3], + "idx1": [0, 0, 1, 1, 0, 1, 0, 0, 1, 2], + "idx2": [0, 1, 0, 1, 0, 0, 0, 1, 0, 0], + "val2": [ + "to the even", + " regular platelets", + " regular", + " ironic epitaphs nag e", + "l accounts", + " blithely ironic theodolites integrate boldly: caref", + " deposits eat slyly ironic", + " even instructions", + " express foxes detect slyly", + " blithely even accounts abov", + ], + } + ), + "explode_07", + order_sensitive=True, + skipped_dialects={"ANSI", "SQLITE"}, + ), + id="explode_07", + ), pytest.param( PyDoughPandasTest( "result = (" @@ -4513,7 +4550,7 @@ " .EXPLODE(val1, 'exp2', value_name='val2', index_name='idx2', version='string', delimiter=' ')" " .EXPLODE(val2, 'exp3', value_name='val3', index_name='idx3', version='string', delimiter=',')" " .WHERE(val3 != '')" - " .CALCULATE(idx1, idx2, idx3, val3)" + " .CALCULATE(key, idx1, idx2, idx3, val3)" " .ORDER_BY(key.ASC(), idx1.ASC(), idx2.ASC(), idx3.ASC())" ")", "TPCH", @@ -4521,27 +4558,31 @@ { "key": [1] * 10 + [2] * 8 + [3] * 14, "idx1": [ + 0, + 0, + 0, + 0, + 0, 1, 1, 1, 1, 1, - 2, - 2, - 2, - 2, - 2, + 0, + 0, + 1, 1, 1, - 2, - 2, - 2, - 2, - 2, - 2, 1, 1, 1, + 0, + 0, + 0, + 0, + 0, + 0, + 1, 1, 1, 1, @@ -4549,24 +4590,20 @@ 2, 2, 2, - 3, - 3, - 3, - 3, ], "idx2": [ + 0, 1, 2, 3, 4, - 5, 1, 2, 3, 4, 5, + 0, 1, - 2, 1, 2, 3, @@ -4588,7 +4625,7 @@ 3, 4, ], - "idx3": [1] * 32, + "idx3": [0] * 32, "val3": [ "to", "the", @@ -4625,11 +4662,11 @@ ], } ), - "explode_07", + "explode_08", order_sensitive=True, skipped_dialects={"ANSI", "SQLITE"}, ), - id="explode_07", + id="explode_08", ), pytest.param( PyDoughPandasTest( diff --git a/tests/test_plan_refsols/explode_07.txt b/tests/test_plan_refsols/explode_07.txt index ddc0890f1..64d1850e4 100644 --- a/tests/test_plan_refsols/explode_07.txt +++ b/tests/test_plan_refsols/explode_07.txt @@ -1,7 +1,5 @@ -ROOT(columns=[('idx1', idx1), ('idx2', idx2), ('idx3', idx3), ('val3', val3)], orderings=[(key):asc_first, (idx1):asc_first, (idx2):asc_first, (idx3):asc_first]) - EXPLODE(val2, value_name='val3', index_name='idx3', version='string', delimiter=',', filtering=True, is_distinct=False, columns={'idx1': idx1, 'idx2': idx2, 'idx3': idx3, 'key': key, 'val3': val3}) - EXPLODE(val1, value_name='val2', index_name='idx2', version='string', delimiter=' ', filtering=True, is_distinct=False, columns={'idx1': idx1, 'idx2': idx2, 'key': key, 'val2': val2}) - EXPLODE(comment, value_name='val1', index_name='idx1', version='string', delimiter='.', filtering=True, is_distinct=False, columns={'idx1': idx1, 'key': key, 'val1': val1}) - FILTER(condition=val3 != '':string, columns={'comment': comment, 'key': key}) - LIMIT(limit=3:numeric, columns={'comment': comment, 'key': key}, orderings=[(key):asc_first]) - SCAN(table=tpch.CUSTOMER, columns={'comment': c_comment, 'key': c_custkey}) +ROOT(columns=[('key', key), ('idx1', idx1), ('idx2', idx2), ('val2', val2)], orderings=[(key):asc_first, (idx1):asc_first, (idx2):asc_first]) + EXPLODE(val1, value_name='val2', index_name='idx2', version='string', delimiter=',', filtering=True, is_distinct=False, columns={'idx1': idx1, 'idx2': idx2, 'key': key, 'val2': val2}) + EXPLODE(comment, value_name='val1', index_name='idx1', version='string', delimiter='.', filtering=True, is_distinct=False, columns={'idx1': idx1, 'key': key, 'val1': val1}) + LIMIT(limit=3:numeric, columns={'comment': comment, 'key': key}, orderings=[(key):asc_first]) + SCAN(table=tpch.CUSTOMER, columns={'comment': c_comment, 'key': c_custkey}) diff --git a/tests/test_plan_refsols/explode_08.txt b/tests/test_plan_refsols/explode_08.txt new file mode 100644 index 000000000..4c9c4641a --- /dev/null +++ b/tests/test_plan_refsols/explode_08.txt @@ -0,0 +1,7 @@ +ROOT(columns=[('key', key), ('idx1', idx1), ('idx2', idx2), ('idx3', idx3), ('val3', val3)], orderings=[(key):asc_first, (idx1):asc_first, (idx2):asc_first, (idx3):asc_first]) + EXPLODE(val2, value_name='val3', index_name='idx3', version='string', delimiter=',', filtering=True, is_distinct=False, columns={'idx1': idx1, 'idx2': idx2, 'idx3': idx3, 'key': key, 'val3': val3}) + EXPLODE(val1, value_name='val2', index_name='idx2', version='string', delimiter=' ', filtering=True, is_distinct=False, columns={'idx1': idx1, 'idx2': idx2, 'key': key, 'val2': val2}) + EXPLODE(comment, value_name='val1', index_name='idx1', version='string', delimiter='.', filtering=True, is_distinct=False, columns={'idx1': idx1, 'key': key, 'val1': val1}) + FILTER(condition=val3 != '':string, columns={'comment': comment, 'key': key}) + LIMIT(limit=3:numeric, columns={'comment': comment, 'key': key}, orderings=[(key):asc_first]) + SCAN(table=tpch.CUSTOMER, columns={'comment': c_comment, 'key': c_custkey}) diff --git a/tests/test_sql_refsols/explode_01_postgres.sql b/tests/test_sql_refsols/explode_01_postgres.sql index 853bb781a..cb13f01ba 100644 --- a/tests/test_sql_refsols/explode_01_postgres.sql +++ b/tests/test_sql_refsols/explode_01_postgres.sql @@ -9,12 +9,12 @@ WITH _s1 AS ( SELECT region.r_name AS region_name, _s1.nation_names, - _l.idx - 1 AS nation_idx, - _l.val AS nation_name + _s2.idx - 1 AS nation_idx, + _s2.val AS nation_name FROM tpch.region AS region JOIN _s1 AS _s1 ON _s1.region_key = region.r_regionkey -CROSS JOIN LATERAL UNNEST(_s1.nation_names) WITH ORDINALITY AS _l(val, idx) +CROSS JOIN LATERAL UNNEST(_s1.nation_names) WITH ORDINALITY AS _s2(val, idx) ORDER BY 1 NULLS FIRST, 3 NULLS FIRST diff --git a/tests/test_sql_refsols/explode_01_snowflake.sql b/tests/test_sql_refsols/explode_01_snowflake.sql index aa547d6c7..cbbfc8b8d 100644 --- a/tests/test_sql_refsols/explode_01_snowflake.sql +++ b/tests/test_sql_refsols/explode_01_snowflake.sql @@ -9,12 +9,12 @@ WITH _s1 AS ( SELECT region.r_name AS region_name, _s1.nation_names, - _l.index AS nation_idx, - _l.value AS nation_name + _s2.index AS nation_idx, + _s2.value AS nation_name FROM tpch.region AS region JOIN _s1 AS _s1 ON _s1.region_key = region.r_regionkey -CROSS JOIN LATERAL FLATTEN(_s1.nation_names) AS _l(seq, key, path, index, value, this) +CROSS JOIN LATERAL FLATTEN(_s1.nation_names) AS _s2(seq, key, path, index, value, this) ORDER BY 1 NULLS FIRST, 3 NULLS FIRST diff --git a/tests/test_sql_refsols/explode_02_postgres.sql b/tests/test_sql_refsols/explode_02_postgres.sql index f733a5219..2c2606bb2 100644 --- a/tests/test_sql_refsols/explode_02_postgres.sql +++ b/tests/test_sql_refsols/explode_02_postgres.sql @@ -1,8 +1,8 @@ SELECT tbl.key, tbl.arr, - _l.idx - 1 AS arr_idx, - _l.val AS arr_val + _s0.idx - 1 AS arr_idx, + _s0.val AS arr_val FROM (VALUES ('A', ARRAY[1]), ('B', ( @@ -10,7 +10,7 @@ FROM (VALUES )[1 : 0]), ('C', ARRAY[2, 3, NULL, 4]), ('D', ARRAY[5, 6])) AS tbl(key, arr) -CROSS JOIN LATERAL UNNEST(tbl.arr) WITH ORDINALITY AS _l(val, idx) +CROSS JOIN LATERAL UNNEST(tbl.arr) WITH ORDINALITY AS _s0(val, idx) ORDER BY 1 NULLS FIRST, 3 NULLS FIRST diff --git a/tests/test_sql_refsols/explode_03_postgres.sql b/tests/test_sql_refsols/explode_03_postgres.sql index c2c1cac48..f73b651fc 100644 --- a/tests/test_sql_refsols/explode_03_postgres.sql +++ b/tests/test_sql_refsols/explode_03_postgres.sql @@ -7,10 +7,10 @@ WITH _q_0 AS ( LIMIT 5 ) SELECT - _l.val, - _l.idx - 1 AS idx + _s0.val, + _s0.idx - 1 AS idx FROM _q_0 AS _q_0 -CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_q_0.name, '#')) WITH ORDINALITY AS _l(val, idx) +CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_q_0.name, '#')) WITH ORDINALITY AS _s0(val, idx) ORDER BY _q_0.name NULLS FIRST, 2 NULLS FIRST diff --git a/tests/test_sql_refsols/explode_03_snowflake.sql b/tests/test_sql_refsols/explode_03_snowflake.sql index 67a868dc1..0e33b7a7a 100644 --- a/tests/test_sql_refsols/explode_03_snowflake.sql +++ b/tests/test_sql_refsols/explode_03_snowflake.sql @@ -7,10 +7,10 @@ WITH _q_0 AS ( LIMIT 5 ) SELECT - _l.value AS val, - _l.index - 1 AS idx + _s0.value AS val, + _s0.index - 1 AS idx FROM _q_0 AS _q_0 -CROSS JOIN LATERAL SPLIT_TO_TABLE(_q_0.name, '#') AS _l +CROSS JOIN LATERAL SPLIT_TO_TABLE(_q_0.name, '#') AS _s0 ORDER BY _q_0.name NULLS FIRST, 2 NULLS FIRST diff --git a/tests/test_sql_refsols/explode_04_postgres.sql b/tests/test_sql_refsols/explode_04_postgres.sql index c93c07450..76cf0a50c 100644 --- a/tests/test_sql_refsols/explode_04_postgres.sql +++ b/tests/test_sql_refsols/explode_04_postgres.sql @@ -3,42 +3,42 @@ WITH _q_0 AS ( r_regionkey AS key, r_name AS name FROM tpch.region -), _s1 AS ( +), _s2 AS ( SELECT _q_0.key, COUNT(*) AS n_rows FROM _q_0 AS _q_0 - CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_q_0.name, 'E')) WITH ORDINALITY AS _l(val, idx) + CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_q_0.name, 'E')) WITH ORDINALITY AS _s0(val, idx) GROUP BY 1 -), _s3 AS ( +), _s5 AS ( SELECT _q_1.key, COUNT(*) AS n_rows FROM _q_0 AS _q_1 - CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_q_1.name, 'I')) WITH ORDINALITY AS _l(val, idx) + CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_q_1.name, 'I')) WITH ORDINALITY AS _s3(val, idx) GROUP BY 1 -), _s5 AS ( +), _s8 AS ( SELECT _q_2.key, COUNT(*) AS n_rows FROM _q_0 AS _q_2 - CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_q_2.name, ' ')) WITH ORDINALITY AS _l(val, idx) + CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_q_2.name, ' ')) WITH ORDINALITY AS _s6(val, idx) GROUP BY 1 ) SELECT region.r_name AS region_name, - COALESCE(_s1.n_rows, 0) AS n_e_chunks, - COALESCE(_s3.n_rows, 0) AS n_i_chunks, - COALESCE(_s5.n_rows, 0) AS n_space_chunks + COALESCE(_s2.n_rows, 0) AS n_e_chunks, + COALESCE(_s5.n_rows, 0) AS n_i_chunks, + COALESCE(_s8.n_rows, 0) AS n_space_chunks FROM tpch.region AS region -LEFT JOIN _s1 AS _s1 - ON _s1.key = region.r_regionkey -LEFT JOIN _s3 AS _s3 - ON _s3.key = region.r_regionkey +LEFT JOIN _s2 AS _s2 + ON _s2.key = region.r_regionkey LEFT JOIN _s5 AS _s5 ON _s5.key = region.r_regionkey +LEFT JOIN _s8 AS _s8 + ON _s8.key = region.r_regionkey ORDER BY 1 NULLS FIRST diff --git a/tests/test_sql_refsols/explode_04_snowflake.sql b/tests/test_sql_refsols/explode_04_snowflake.sql index 79267f1d8..e357a1660 100644 --- a/tests/test_sql_refsols/explode_04_snowflake.sql +++ b/tests/test_sql_refsols/explode_04_snowflake.sql @@ -3,42 +3,42 @@ WITH _q_0 AS ( r_regionkey AS key, r_name AS name FROM tpch.region -), _s1 AS ( +), _s2 AS ( SELECT _q_0.key, COUNT(*) AS n_rows FROM _q_0 AS _q_0 - CROSS JOIN LATERAL SPLIT_TO_TABLE(_q_0.name, 'E') AS _l + CROSS JOIN LATERAL SPLIT_TO_TABLE(_q_0.name, 'E') AS _s0 GROUP BY 1 -), _s3 AS ( +), _s5 AS ( SELECT _q_1.key, COUNT(*) AS n_rows FROM _q_0 AS _q_1 - CROSS JOIN LATERAL SPLIT_TO_TABLE(_q_1.name, 'I') AS _l + CROSS JOIN LATERAL SPLIT_TO_TABLE(_q_1.name, 'I') AS _s3 GROUP BY 1 -), _s5 AS ( +), _s8 AS ( SELECT _q_2.key, COUNT(*) AS n_rows FROM _q_0 AS _q_2 - CROSS JOIN LATERAL SPLIT_TO_TABLE(_q_2.name, ' ') AS _l + CROSS JOIN LATERAL SPLIT_TO_TABLE(_q_2.name, ' ') AS _s6 GROUP BY 1 ) SELECT region.r_name AS region_name, - COALESCE(_s1.n_rows, 0) AS n_e_chunks, - COALESCE(_s3.n_rows, 0) AS n_i_chunks, - COALESCE(_s5.n_rows, 0) AS n_space_chunks + COALESCE(_s2.n_rows, 0) AS n_e_chunks, + COALESCE(_s5.n_rows, 0) AS n_i_chunks, + COALESCE(_s8.n_rows, 0) AS n_space_chunks FROM tpch.region AS region -LEFT JOIN _s1 AS _s1 - ON _s1.key = region.r_regionkey -LEFT JOIN _s3 AS _s3 - ON _s3.key = region.r_regionkey +LEFT JOIN _s2 AS _s2 + ON _s2.key = region.r_regionkey LEFT JOIN _s5 AS _s5 ON _s5.key = region.r_regionkey +LEFT JOIN _s8 AS _s8 + ON _s8.key = region.r_regionkey ORDER BY 1 NULLS FIRST diff --git a/tests/test_sql_refsols/explode_07_postgres.sql b/tests/test_sql_refsols/explode_07_postgres.sql new file mode 100644 index 000000000..23e2ce3ba --- /dev/null +++ b/tests/test_sql_refsols/explode_07_postgres.sql @@ -0,0 +1,21 @@ +WITH _q_0 AS ( + SELECT + c_comment AS comment, + c_custkey AS key + FROM tpch.customer + ORDER BY + 2 NULLS FIRST + LIMIT 3 +) +SELECT + _q_0.key, + _s0.idx - 1 AS idx1, + _s1.idx - 1 AS idx2, + _s1.val AS val2 +FROM _q_0 AS _q_0 +CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) +CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx) +ORDER BY + 1 NULLS FIRST, + 2 NULLS FIRST, + 3 NULLS FIRST diff --git a/tests/test_sql_refsols/explode_07_snowflake.sql b/tests/test_sql_refsols/explode_07_snowflake.sql new file mode 100644 index 000000000..9d66b8cc6 --- /dev/null +++ b/tests/test_sql_refsols/explode_07_snowflake.sql @@ -0,0 +1,21 @@ +WITH _q_0 AS ( + SELECT + c_comment AS comment, + c_custkey AS key + FROM tpch.customer + ORDER BY + 2 NULLS FIRST + LIMIT 3 +) +SELECT + _q_0.key, + _s0.index - 1 AS idx1, + _s1.index - 1 AS idx2, + _s1.value AS val2 +FROM _q_0 AS _q_0 +CROSS JOIN LATERAL SPLIT_TO_TABLE(_q_0.comment, '.') AS _s0 +CROSS JOIN LATERAL SPLIT_TO_TABLE(_s0.value, ',') AS _s1 +ORDER BY + 1 NULLS FIRST, + 2 NULLS FIRST, + 3 NULLS FIRST diff --git a/tests/test_sql_refsols/explode_08_postgres.sql b/tests/test_sql_refsols/explode_08_postgres.sql new file mode 100644 index 000000000..7e847ba2d --- /dev/null +++ b/tests/test_sql_refsols/explode_08_postgres.sql @@ -0,0 +1,26 @@ +WITH _t1 AS ( + SELECT + c_comment AS comment, + c_custkey AS key + FROM tpch.customer + ORDER BY + 2 NULLS FIRST + LIMIT 3 +) +SELECT + _t1.key, + _s0.idx - 1 AS idx1, + _s1.idx - 1 AS idx2, + _s2.idx - 1 AS idx3, + _s2.val AS val3 +FROM _t1 AS _t1 +CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_t1.comment, '.')) WITH ORDINALITY AS _s0(val, idx) +CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s0.val, ' ')) WITH ORDINALITY AS _s1(val, idx) +CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s1.val, ',')) WITH ORDINALITY AS _s2(val, idx) +WHERE + val3 <> '' +ORDER BY + 1 NULLS FIRST, + 2 NULLS FIRST, + 3 NULLS FIRST, + 4 NULLS FIRST diff --git a/tests/test_sql_refsols/explode_08_snowflake.sql b/tests/test_sql_refsols/explode_08_snowflake.sql new file mode 100644 index 000000000..d34875a20 --- /dev/null +++ b/tests/test_sql_refsols/explode_08_snowflake.sql @@ -0,0 +1,26 @@ +WITH _t1 AS ( + SELECT + c_comment AS comment, + c_custkey AS key + FROM tpch.customer + ORDER BY + 2 NULLS FIRST + LIMIT 3 +) +SELECT + _t1.key, + _s0.index - 1 AS idx1, + _s1.index - 1 AS idx2, + _s2.index - 1 AS idx3, + _s2.value AS val3 +FROM _t1 AS _t1 +CROSS JOIN LATERAL SPLIT_TO_TABLE(_t1.comment, '.') AS _s0 +CROSS JOIN LATERAL SPLIT_TO_TABLE(_s0.value, ' ') AS _s1 +CROSS JOIN LATERAL SPLIT_TO_TABLE(_s1.value, ',') AS _s2 +WHERE + val3 <> '' +ORDER BY + 1 NULLS FIRST, + 2 NULLS FIRST, + 3 NULLS FIRST, + 4 NULLS FIRST From cea2df7dc97a1828545805c7279361598b9c41a1 Mon Sep 17 00:00:00 2001 From: knassre-bodo Date: Fri, 7 Aug 2026 10:56:43 -0700 Subject: [PATCH 24/44] Added more tests --- tests/test_pipeline_tpch_custom.py | 125 ++++++++++++++++++ tests/test_plan_refsols/explode_09.txt | 4 + tests/test_plan_refsols/explode_10.txt | 5 + .../test_sql_refsols/explode_09_postgres.sql | 10 ++ .../test_sql_refsols/explode_09_snowflake.sql | 10 ++ .../test_sql_refsols/explode_10_postgres.sql | 13 ++ .../test_sql_refsols/explode_10_snowflake.sql | 16 +++ 7 files changed, 183 insertions(+) create mode 100644 tests/test_plan_refsols/explode_09.txt create mode 100644 tests/test_plan_refsols/explode_10.txt create mode 100644 tests/test_sql_refsols/explode_09_postgres.sql create mode 100644 tests/test_sql_refsols/explode_09_snowflake.sql create mode 100644 tests/test_sql_refsols/explode_10_postgres.sql create mode 100644 tests/test_sql_refsols/explode_10_snowflake.sql diff --git a/tests/test_pipeline_tpch_custom.py b/tests/test_pipeline_tpch_custom.py index 5e76212a3..2a04e4eec 100644 --- a/tests/test_pipeline_tpch_custom.py +++ b/tests/test_pipeline_tpch_custom.py @@ -4668,6 +4668,131 @@ ), id="explode_08", ), + pytest.param( + PyDoughPandasTest( + "result = (" + " regions" + " .CALCULATE(name)" + " .EXPLODE(comment, 'words', value_name='val', index_name='idx', version='string', delimiter=' ')" + " .PARTITION(name='region_partition', by=name)" + " .CALCULATE(name, n_words=COUNT(words), words_list=LISTOF(words.val))" + " .ORDER_BY(name)" + ")", + "TPCH", + lambda: pd.DataFrame( + { + "name": ["AFRICA", "AMERICA", "ASIA", "EUROPE", "MIDDLE EAST"], + "n_words": [17, 6, 6, 7, 13], + "words_list": [ + [ + "lar", + "deposits.", + "blithely", + "final", + "packages", + "cajole.", + "regular", + "waters", + "are", + "final", + "requests.", + "regular", + "accounts", + "are", + "according", + "to", + "", + ], + ["hs", "use", "ironic,", "even", "requests.", "s"], + ["ges.", "thinly", "even", "pinto", "beans", "ca"], + [ + "ly", + "final", + "courts", + "cajole", + "furiously", + "final", + "excuse", + ], + [ + "uickly", + "special", + "accounts", + "cajole", + "carefully", + "blithely", + "close", + "requests.", + "carefully", + "final", + "asymptotes", + "haggle", + "furiousl", + ], + ], + } + ), + "explode_09", + order_sensitive=True, + skipped_dialects={"ANSI", "SQLITE", "MYSQL"}, + ), + id="explode_09", + ), + pytest.param( + PyDoughPandasTest( + "result = (" + " customers" + " .CALCULATE(cleaned_comment=STRIP(REPLACE(REPLACE(REPLACE(REPLACE(comment, ';', ''), ',', ''), ':', ''), '.', ''), ' '))" + " .EXPLODE(cleaned_comment, 'words', value_name='val', index_name='idx', version='string', delimiter=' ')" + " .CALCULATE(word_length=LENGTH(val))" + " .PARTITION(name='lengths', by=word_length)" + " .CALCULATE(word_length, n_words=COUNT(words), n_unique_words=NDISTINCT(words.val))" + " .ORDER_BY(word_length)" + ")", + "TPCH", + lambda: pd.DataFrame( + { + "word_length": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], + "n_words": [ + 40111, + 63249, + 180758, + 127312, + 303297, + 172864, + 223795, + 233379, + 141958, + 17193, + 21597, + 26271, + 478, + 168, + ], + "n_unique_words": [ + 27, + 152, + 298, + 363, + 346, + 282, + 226, + 159, + 112, + 66, + 38, + 21, + 11, + 2, + ], + } + ), + "explode_10", + order_sensitive=True, + skipped_dialects={"ANSI", "SQLITE"}, + ), + id="explode_10", + ), pytest.param( PyDoughPandasTest( simple_range_1, diff --git a/tests/test_plan_refsols/explode_09.txt b/tests/test_plan_refsols/explode_09.txt new file mode 100644 index 000000000..dec8a054f --- /dev/null +++ b/tests/test_plan_refsols/explode_09.txt @@ -0,0 +1,4 @@ +ROOT(columns=[('name', name), ('n_words', n_rows), ('words_list', listof_val)], orderings=[(name):asc_first]) + AGGREGATE(keys={'name': name}, aggregations={'listof_val': LISTOF(val), 'n_rows': COUNT()}) + EXPLODE(comment, value_name='val', index_name='idx', version='string', delimiter=' ', filtering=True, is_distinct=False, columns={'idx': idx, 'name': name, 'val': val}) + SCAN(table=tpch.REGION, columns={'comment': r_comment, 'name': r_name}) diff --git a/tests/test_plan_refsols/explode_10.txt b/tests/test_plan_refsols/explode_10.txt new file mode 100644 index 000000000..3b3c8df04 --- /dev/null +++ b/tests/test_plan_refsols/explode_10.txt @@ -0,0 +1,5 @@ +ROOT(columns=[('word_length', length_val), ('n_words', n_rows), ('n_unique_words', ndistinct_val)], orderings=[(length_val):asc_first]) + AGGREGATE(keys={'length_val': LENGTH(val)}, aggregations={'n_rows': COUNT(), 'ndistinct_val': NDISTINCT(val)}) + EXPLODE(cleaned_comment, value_name='val', index_name='idx', version='string', delimiter=' ', filtering=True, is_distinct=False, columns={'idx': idx, 'val': val}) + PROJECT(columns={'cleaned_comment': STRIP(REPLACE(REPLACE(REPLACE(REPLACE(comment, ';':string, '':string), ',':string, '':string), ':':string, '':string), '.':string, '':string), ' ':string)}) + SCAN(table=tpch.CUSTOMER, columns={'comment': c_comment}) diff --git a/tests/test_sql_refsols/explode_09_postgres.sql b/tests/test_sql_refsols/explode_09_postgres.sql new file mode 100644 index 000000000..25b9e7668 --- /dev/null +++ b/tests/test_sql_refsols/explode_09_postgres.sql @@ -0,0 +1,10 @@ +SELECT + region.r_name AS name, + COUNT(*) AS n_words, + ARRAY_AGG(_s0.val) AS words_list +FROM tpch.region AS region +CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(region.r_comment, ' ')) WITH ORDINALITY AS _s0(val, idx) +GROUP BY + 1 +ORDER BY + 1 NULLS FIRST diff --git a/tests/test_sql_refsols/explode_09_snowflake.sql b/tests/test_sql_refsols/explode_09_snowflake.sql new file mode 100644 index 000000000..59b1ebf6c --- /dev/null +++ b/tests/test_sql_refsols/explode_09_snowflake.sql @@ -0,0 +1,10 @@ +SELECT + region.r_name AS name, + COUNT(*) AS n_words, + ARRAY_AGG(_s0.value) AS words_list +FROM tpch.region AS region +CROSS JOIN LATERAL SPLIT_TO_TABLE(region.r_comment, ' ') AS _s0 +GROUP BY + 1 +ORDER BY + 1 NULLS FIRST diff --git a/tests/test_sql_refsols/explode_10_postgres.sql b/tests/test_sql_refsols/explode_10_postgres.sql new file mode 100644 index 000000000..12c940f60 --- /dev/null +++ b/tests/test_sql_refsols/explode_10_postgres.sql @@ -0,0 +1,13 @@ +SELECT + LENGTH(_s0.val) AS word_length, + COUNT(*) AS n_words, + COUNT(DISTINCT _s0.val) AS n_unique_words +FROM tpch.customer AS customer +CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY( + TRIM(' ' FROM REPLACE(REPLACE(REPLACE(REPLACE(customer.c_comment, ';', ''), ',', ''), ':', ''), '.', '')), + ' ' +)) WITH ORDINALITY AS _s0(val, idx) +GROUP BY + 1 +ORDER BY + 1 NULLS FIRST diff --git a/tests/test_sql_refsols/explode_10_snowflake.sql b/tests/test_sql_refsols/explode_10_snowflake.sql new file mode 100644 index 000000000..07bed5c1f --- /dev/null +++ b/tests/test_sql_refsols/explode_10_snowflake.sql @@ -0,0 +1,16 @@ +SELECT + LENGTH(_s0.value) AS word_length, + COUNT(*) AS n_words, + COUNT(DISTINCT _s0.value) AS n_unique_words +FROM tpch.customer AS customer +CROSS JOIN LATERAL SPLIT_TO_TABLE( + TRIM( + REPLACE(REPLACE(REPLACE(REPLACE(customer.c_comment, ';', ''), ',', ''), ':', ''), '.', ''), + ' ' + ), + ' ' +) AS _s0 +GROUP BY + 1 +ORDER BY + 1 NULLS FIRST From 246279108376ff3bae863f7357dc2a88b35644d8 Mon Sep 17 00:00:00 2001 From: knassre-bodo Date: Fri, 7 Aug 2026 14:30:14 -0700 Subject: [PATCH 25/44] Added more tests, fixed filter pushdown bugs --- pydough/conversion/filter_pushdown.py | 22 +++++++ tests/test_pipeline_tpch_custom.py | 63 +++++++++++++++++++ tests/test_plan_refsols/explode_08.txt | 8 +-- tests/test_plan_refsols/explode_11.txt | 10 +++ tests/test_plan_refsols/explode_12.txt | 3 + .../test_sql_refsols/explode_08_postgres.sql | 13 ++-- .../test_sql_refsols/explode_08_snowflake.sql | 13 ++-- .../test_sql_refsols/explode_11_postgres.sql | 26 ++++++++ .../test_sql_refsols/explode_11_snowflake.sql | 32 ++++++++++ .../test_sql_refsols/explode_12_postgres.sql | 11 ++++ 10 files changed, 183 insertions(+), 18 deletions(-) create mode 100644 tests/test_plan_refsols/explode_11.txt create mode 100644 tests/test_plan_refsols/explode_12.txt create mode 100644 tests/test_sql_refsols/explode_11_postgres.sql create mode 100644 tests/test_sql_refsols/explode_11_snowflake.sql create mode 100644 tests/test_sql_refsols/explode_12_postgres.sql diff --git a/pydough/conversion/filter_pushdown.py b/pydough/conversion/filter_pushdown.py index 273782dfc..881b39794 100644 --- a/pydough/conversion/filter_pushdown.py +++ b/pydough/conversion/filter_pushdown.py @@ -12,6 +12,7 @@ CallExpression, ColumnReference, EmptySingleton, + Explode, Filter, GeneratedTable, Join, @@ -319,6 +320,27 @@ def visit_generated_table(self, generated_table: GeneratedTable) -> RelationalNo # cannot be pushed down any further. return self.flush_remaining_filters(generated_table, self.filters, set()) + def visit_explode(self, explode: Explode) -> RelationalNode: + pushable_filters: set[RelationalExpression] + remaining_filters: set[RelationalExpression] + # Push all filters that only depend on columns that pass through + # from the input, as opposed to being generated outputs. + allowed_cols: set[str] = set() + for name, expr in explode.columns.items(): + if not ( + isinstance(expr, ColumnReference) + and expr.name + in (explode.explode_spec.value_name, explode.explode_spec.index_name) + ): + allowed_cols.add(name) + pushable_filters, remaining_filters = partition_expressions( + self.filters, + lambda expr: only_references_columns(expr, allowed_cols), + ) + return self.flush_remaining_filters( + explode, remaining_filters, pushable_filters + ) + def push_filters(node: RelationalNode, session: PyDoughSession) -> RelationalNode: """ diff --git a/tests/test_pipeline_tpch_custom.py b/tests/test_pipeline_tpch_custom.py index 2a04e4eec..f5db6b220 100644 --- a/tests/test_pipeline_tpch_custom.py +++ b/tests/test_pipeline_tpch_custom.py @@ -4793,6 +4793,69 @@ ), id="explode_10", ), + pytest.param( + PyDoughPandasTest( + "supplier_words = (" + " suppliers" + " .CALCULATE(supp_comment=STRIP(REPLACE(REPLACE(REPLACE(REPLACE(comment, ';', ''), ',', ''), ':', ''), '.', ''), ' '))" + " .EXPLODE(supp_comment, 'supp_words', value_name='supp_word', index_name='supp_idx', version='string', delimiter=' ')" + " .PARTITION(name='s_words', by=supp_word)" + " .CALCULATE(supp_word)" + ")\n" + "customer_words = (" + " customers" + " .CALCULATE(cust_comment=STRIP(REPLACE(REPLACE(REPLACE(REPLACE(comment, ';', ''), ',', ''), ':', ''), '.', ''), ' '))" + " .EXPLODE(cust_comment, 'cust_words', value_name='cust_word', index_name='cust_idx', version='string', delimiter=' ')" + " .PARTITION(name='c_words', by=cust_word)" + " .CALCULATE(cust_word)" + ")\n" + "match_words = (" + " supplier_words" + " .WHERE(HAS(CROSS(customer_words).WHERE(supp_word == cust_word)))" + ")\n" + "result = TPCH.CALCULATE(n_double_words=COUNT(match_words))", + "TPCH", + lambda: pd.DataFrame( + { + "n_double_words": [1249], + } + ), + "explode_11", + order_sensitive=True, + skipped_dialects={"ANSI", "SQLITE"}, + ), + id="explode_11", + ), + pytest.param( + PyDoughPandasTest( + "array_data = pydough.dataframe_collection(name='tbl', dataframe=array_df, unique_column_names=['key'])\n" + "result = (" + " array_data" + " .CALCULATE(key)" + " .EXPLODE(arr, 'exploded_array', value_name='arr_val', version='array', filtering=True, is_distinct=True)" + " .PARTITION(name='groups', by=(key, arr_val))" + " .CALCULATE(key, arr_val)" + ")", + "TPCH", + lambda: pd.DataFrame( + { + "key": ["A", "C", "C", "C", "C", "D", "D"], + "arr_val": [1, 2, 3, None, 4, 5, 6], + } + ), + "explode_12", + skipped_dialects={"ANSI", "SQLITE", "SNOWFLAKE"}, + kwargs={ + "array_df": pd.DataFrame( + { + "key": ["A", "B", "C", "D"], + "arr": [[1], [], [2, 3, None, 4], [5, 6]], + } + ) + }, + ), + id="explode_12", + ), pytest.param( PyDoughPandasTest( simple_range_1, diff --git a/tests/test_plan_refsols/explode_08.txt b/tests/test_plan_refsols/explode_08.txt index 4c9c4641a..bcb2721b9 100644 --- a/tests/test_plan_refsols/explode_08.txt +++ b/tests/test_plan_refsols/explode_08.txt @@ -1,7 +1,7 @@ ROOT(columns=[('key', key), ('idx1', idx1), ('idx2', idx2), ('idx3', idx3), ('val3', val3)], orderings=[(key):asc_first, (idx1):asc_first, (idx2):asc_first, (idx3):asc_first]) - EXPLODE(val2, value_name='val3', index_name='idx3', version='string', delimiter=',', filtering=True, is_distinct=False, columns={'idx1': idx1, 'idx2': idx2, 'idx3': idx3, 'key': key, 'val3': val3}) - EXPLODE(val1, value_name='val2', index_name='idx2', version='string', delimiter=' ', filtering=True, is_distinct=False, columns={'idx1': idx1, 'idx2': idx2, 'key': key, 'val2': val2}) - EXPLODE(comment, value_name='val1', index_name='idx1', version='string', delimiter='.', filtering=True, is_distinct=False, columns={'idx1': idx1, 'key': key, 'val1': val1}) - FILTER(condition=val3 != '':string, columns={'comment': comment, 'key': key}) + FILTER(condition=val3 != '':string, columns={'idx1': idx1, 'idx2': idx2, 'idx3': idx3, 'key': key, 'val3': val3}) + EXPLODE(val2, value_name='val3', index_name='idx3', version='string', delimiter=',', filtering=True, is_distinct=False, columns={'idx1': idx1, 'idx2': idx2, 'idx3': idx3, 'key': key, 'val3': val3}) + EXPLODE(val1, value_name='val2', index_name='idx2', version='string', delimiter=' ', filtering=True, is_distinct=False, columns={'idx1': idx1, 'idx2': idx2, 'key': key, 'val2': val2}) + EXPLODE(comment, value_name='val1', index_name='idx1', version='string', delimiter='.', filtering=True, is_distinct=False, columns={'idx1': idx1, 'key': key, 'val1': val1}) LIMIT(limit=3:numeric, columns={'comment': comment, 'key': key}, orderings=[(key):asc_first]) SCAN(table=tpch.CUSTOMER, columns={'comment': c_comment, 'key': c_custkey}) diff --git a/tests/test_plan_refsols/explode_11.txt b/tests/test_plan_refsols/explode_11.txt new file mode 100644 index 000000000..ff6212ea6 --- /dev/null +++ b/tests/test_plan_refsols/explode_11.txt @@ -0,0 +1,10 @@ +ROOT(columns=[('n_double_words', ndistinct_supp_word)], orderings=[]) + AGGREGATE(keys={}, aggregations={'ndistinct_supp_word': NDISTINCT(supp_word)}) + JOIN(condition=t0.supp_word == t1.cust_word, type=SEMI, columns={'supp_word': t0.supp_word}) + EXPLODE(supp_comment, value_name='supp_word', index_name='supp_idx', version='string', delimiter=' ', filtering=True, is_distinct=False, columns={'supp_idx': supp_idx, 'supp_word': supp_word}) + PROJECT(columns={'supp_comment': STRIP(REPLACE(REPLACE(REPLACE(REPLACE(comment, ';':string, '':string), ',':string, '':string), ':':string, '':string), '.':string, '':string), ' ':string)}) + SCAN(table=tpch.SUPPLIER, columns={'comment': s_comment}) + AGGREGATE(keys={'cust_word': cust_word}, aggregations={}) + EXPLODE(cust_comment, value_name='cust_word', index_name='cust_idx', version='string', delimiter=' ', filtering=True, is_distinct=False, columns={'cust_idx': cust_idx, 'cust_word': cust_word}) + PROJECT(columns={'cust_comment': STRIP(REPLACE(REPLACE(REPLACE(REPLACE(comment, ';':string, '':string), ',':string, '':string), ':':string, '':string), '.':string, '':string), ' ':string)}) + SCAN(table=tpch.CUSTOMER, columns={'comment': c_comment}) diff --git a/tests/test_plan_refsols/explode_12.txt b/tests/test_plan_refsols/explode_12.txt new file mode 100644 index 000000000..f469babe0 --- /dev/null +++ b/tests/test_plan_refsols/explode_12.txt @@ -0,0 +1,3 @@ +ROOT(columns=[('key', key), ('arr_val', arr_val)], orderings=[]) + EXPLODE(arr, value_name='arr_val', version='array', filtering=True, is_distinct=True, columns={'arr_val': arr_val, 'key': key}) + GENERATED_TABLE(DataframeCollection(name='tbl', shape=(4, 2), columns=['key', 'arr'])) diff --git a/tests/test_sql_refsols/explode_08_postgres.sql b/tests/test_sql_refsols/explode_08_postgres.sql index 7e847ba2d..76acc5249 100644 --- a/tests/test_sql_refsols/explode_08_postgres.sql +++ b/tests/test_sql_refsols/explode_08_postgres.sql @@ -1,4 +1,4 @@ -WITH _t1 AS ( +WITH _q_0 AS ( SELECT c_comment AS comment, c_custkey AS key @@ -8,17 +8,16 @@ WITH _t1 AS ( LIMIT 3 ) SELECT - _t1.key, + _q_0.key, _s0.idx - 1 AS idx1, _s1.idx - 1 AS idx2, _s2.idx - 1 AS idx3, _s2.val AS val3 -FROM _t1 AS _t1 -CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_t1.comment, '.')) WITH ORDINALITY AS _s0(val, idx) +FROM _q_0 AS _q_0 +CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s0.val, ' ')) WITH ORDINALITY AS _s1(val, idx) -CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s1.val, ',')) WITH ORDINALITY AS _s2(val, idx) -WHERE - val3 <> '' +JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s1.val, ',')) WITH ORDINALITY AS _s2(val, idx) + ON _s2.val <> '' ORDER BY 1 NULLS FIRST, 2 NULLS FIRST, diff --git a/tests/test_sql_refsols/explode_08_snowflake.sql b/tests/test_sql_refsols/explode_08_snowflake.sql index d34875a20..07387100a 100644 --- a/tests/test_sql_refsols/explode_08_snowflake.sql +++ b/tests/test_sql_refsols/explode_08_snowflake.sql @@ -1,4 +1,4 @@ -WITH _t1 AS ( +WITH _q_0 AS ( SELECT c_comment AS comment, c_custkey AS key @@ -8,17 +8,16 @@ WITH _t1 AS ( LIMIT 3 ) SELECT - _t1.key, + _q_0.key, _s0.index - 1 AS idx1, _s1.index - 1 AS idx2, _s2.index - 1 AS idx3, _s2.value AS val3 -FROM _t1 AS _t1 -CROSS JOIN LATERAL SPLIT_TO_TABLE(_t1.comment, '.') AS _s0 +FROM _q_0 AS _q_0 +CROSS JOIN LATERAL SPLIT_TO_TABLE(_q_0.comment, '.') AS _s0 CROSS JOIN LATERAL SPLIT_TO_TABLE(_s0.value, ' ') AS _s1 -CROSS JOIN LATERAL SPLIT_TO_TABLE(_s1.value, ',') AS _s2 -WHERE - val3 <> '' +JOIN LATERAL SPLIT_TO_TABLE(_s1.value, ',') AS _s2 + ON _s2.value <> '' ORDER BY 1 NULLS FIRST, 2 NULLS FIRST, diff --git a/tests/test_sql_refsols/explode_11_postgres.sql b/tests/test_sql_refsols/explode_11_postgres.sql new file mode 100644 index 000000000..25b4a35ff --- /dev/null +++ b/tests/test_sql_refsols/explode_11_postgres.sql @@ -0,0 +1,26 @@ +WITH _s3 AS ( + SELECT DISTINCT + _s1.val AS cust_word + FROM tpch.customer AS customer + CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY( + TRIM(' ' FROM REPLACE(REPLACE(REPLACE(REPLACE(customer.c_comment, ';', ''), ',', ''), ':', ''), '.', '')), + ' ' + )) WITH ORDINALITY AS _s1(val, idx) +), _u_0 AS ( + SELECT + cust_word AS _u_1 + FROM _s3 + GROUP BY + 1 +) +SELECT + COUNT(DISTINCT _s0.val) AS n_double_words +FROM tpch.supplier AS supplier +CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY( + TRIM(' ' FROM REPLACE(REPLACE(REPLACE(REPLACE(supplier.s_comment, ';', ''), ',', ''), ':', ''), '.', '')), + ' ' +)) WITH ORDINALITY AS _s0(val, idx) +LEFT JOIN _u_0 AS _u_0 + ON _s0.val = _u_0._u_1 +WHERE + NOT _u_0._u_1 IS NULL diff --git a/tests/test_sql_refsols/explode_11_snowflake.sql b/tests/test_sql_refsols/explode_11_snowflake.sql new file mode 100644 index 000000000..020bd0ace --- /dev/null +++ b/tests/test_sql_refsols/explode_11_snowflake.sql @@ -0,0 +1,32 @@ +WITH _s3 AS ( + SELECT DISTINCT + _s1.value AS cust_word + FROM tpch.customer AS customer + CROSS JOIN LATERAL SPLIT_TO_TABLE( + TRIM( + REPLACE(REPLACE(REPLACE(REPLACE(customer.c_comment, ';', ''), ',', ''), ':', ''), '.', ''), + ' ' + ), + ' ' + ) AS _s1 +), _u_0 AS ( + SELECT + cust_word AS _u_1 + FROM _s3 + GROUP BY + 1 +) +SELECT + COUNT(DISTINCT _s0.value) AS n_double_words +FROM tpch.supplier AS supplier +CROSS JOIN LATERAL SPLIT_TO_TABLE( + TRIM( + REPLACE(REPLACE(REPLACE(REPLACE(supplier.s_comment, ';', ''), ',', ''), ':', ''), '.', ''), + ' ' + ), + ' ' +) AS _s0 +LEFT JOIN _u_0 AS _u_0 + ON _s0.value = _u_0._u_1 +WHERE + NOT _u_0._u_1 IS NULL diff --git a/tests/test_sql_refsols/explode_12_postgres.sql b/tests/test_sql_refsols/explode_12_postgres.sql new file mode 100644 index 000000000..421c31c81 --- /dev/null +++ b/tests/test_sql_refsols/explode_12_postgres.sql @@ -0,0 +1,11 @@ +SELECT + tbl.key, + _s0.val AS arr_val +FROM (VALUES + ('A', ARRAY[1]), + ('B', ( + ARRAY[0] + )[1 : 0]), + ('C', ARRAY[2, 3, NULL, 4]), + ('D', ARRAY[5, 6])) AS tbl(key, arr) +CROSS JOIN LATERAL UNNEST(tbl.arr) WITH ORDINALITY AS _s0(val, idx) From 314c7d41d49b7a5fd87312d208be502a03a10e10 Mon Sep 17 00:00:00 2001 From: knassre-bodo Date: Tue, 11 Aug 2026 10:07:59 -0700 Subject: [PATCH 26/44] Added databricks support --- .../databricks_transform_bindings.py | 76 +++++++++++++++++++ .../postgres_transform_bindings.py | 40 ++++++---- .../explode_01_databricks.sql | 20 +++++ .../explode_02_databricks.sql | 14 ++++ .../explode_03_databricks.sql | 16 ++++ .../explode_04_databricks.sql | 44 +++++++++++ .../explode_07_databricks.sql | 21 +++++ .../explode_08_databricks.sql | 25 ++++++ .../explode_09_databricks.sql | 10 +++ .../explode_10_databricks.sql | 15 ++++ .../explode_11_databricks.sql | 30 ++++++++ .../explode_12_databricks.sql | 9 +++ 12 files changed, 304 insertions(+), 16 deletions(-) create mode 100644 tests/test_sql_refsols/explode_01_databricks.sql create mode 100644 tests/test_sql_refsols/explode_02_databricks.sql create mode 100644 tests/test_sql_refsols/explode_03_databricks.sql create mode 100644 tests/test_sql_refsols/explode_04_databricks.sql create mode 100644 tests/test_sql_refsols/explode_07_databricks.sql create mode 100644 tests/test_sql_refsols/explode_08_databricks.sql create mode 100644 tests/test_sql_refsols/explode_09_databricks.sql create mode 100644 tests/test_sql_refsols/explode_10_databricks.sql create mode 100644 tests/test_sql_refsols/explode_11_databricks.sql create mode 100644 tests/test_sql_refsols/explode_12_databricks.sql diff --git a/pydough/sqlglot/transform_bindings/databricks_transform_bindings.py b/pydough/sqlglot/transform_bindings/databricks_transform_bindings.py index bcf8fec61..7ffc93ace 100644 --- a/pydough/sqlglot/transform_bindings/databricks_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/databricks_transform_bindings.py @@ -8,12 +8,22 @@ from typing import Any import sqlglot.expressions as sqlglot_expressions +from sqlglot.expressions import ( + Explode, + Identifier, + Lateral, + Posexplode, + Select, + Subquery, + TableAlias, +) from sqlglot.expressions import Expression as SQLGlotExpression import pydough.pydough_operators as pydop from pydough.configs import DayOfWeek from pydough.types import NumericType, PyDoughType from pydough.types.boolean_type import BooleanType +from pydough.utilities import ExplodeSpec from .base_transform_bindings import BaseTransformBindings from .sqlglot_transform_utils import DateTimeUnit, apply_parens @@ -113,11 +123,77 @@ def convert_integer( to=sqlglot_expressions.DataType.build("BIGINT"), ) + def generate_dataframe_array_expression( + self, items: list[SQLGlotExpression], inner_type: PyDoughType + ) -> SQLGlotExpression: + return sqlglot_expressions.Array(expressions=items) + def convert_listof( self, args: SQLGlotExpression, types: list[PyDoughType] ) -> SQLGlotExpression: return sqlglot_expressions.ArrayAgg(this=args[0]) + def convert_explode( + self, + input_expr: SQLGlotExpression, + explode_expr: SQLGlotExpression, + explode_spec: ExplodeSpec, + exprs: list[SQLGlotExpression], + val_index: int | None, + idx_index: int | None, + lateral_alias: str, + ) -> SQLGlotExpression: + column_exprs: list[SQLGlotExpression] = [*exprs] + if val_index is not None: + column_exprs[val_index] = sqlglot_expressions.Alias( + this=sqlglot_expressions.Column( + this=sqlglot_expressions.Identifier(this="val"), + table=sqlglot_expressions.Identifier(this=lateral_alias), + ), + alias=sqlglot_expressions.Identifier(this=explode_spec.value_name), + ) + if idx_index is not None and explode_spec.index_name is not None: + column_exprs[idx_index] = sqlglot_expressions.Alias( + this=sqlglot_expressions.Column( + this=sqlglot_expressions.Identifier(this="idx"), + table=sqlglot_expressions.Identifier(this=lateral_alias), + ), + alias=sqlglot_expressions.Identifier(this=explode_spec.index_name), + ) + + if explode_spec.version == "string": + assert explode_spec.delimiter is not None, ( + "Delimiter must be provided for string explode." + ) + explode_expr = sqlglot_expressions.Split( + this=explode_expr, + expression=sqlglot_expressions.Literal.string(explode_spec.delimiter), + ) + explode_op: SQLGlotExpression + lateral_columns: list[SQLGlotExpression] = [ + sqlglot_expressions.Identifier(this="val") + ] + if val_index is None: + explode_op = Explode(this=explode_expr) + else: + explode_op = Posexplode(this=explode_expr) + lateral_columns.insert(0, sqlglot_expressions.Identifier(this="idx")) + result = ( + Select() + .select(*column_exprs) + .from_(Subquery(this=input_expr)) + .join( + Lateral( + this=explode_op, + alias=TableAlias( + this=Identifier(this=lateral_alias), columns=lateral_columns + ), + ) + ) + ) + + return result + def generate_dataframe_item_dialect_expression( self, item: Any, item_type: PyDoughType ) -> SQLGlotExpression: diff --git a/pydough/sqlglot/transform_bindings/postgres_transform_bindings.py b/pydough/sqlglot/transform_bindings/postgres_transform_bindings.py index c90829f71..a21aebc9f 100644 --- a/pydough/sqlglot/transform_bindings/postgres_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/postgres_transform_bindings.py @@ -118,26 +118,34 @@ def convert_explode( sqlglot_expressions.Literal.string(explode_spec.delimiter), ], ) - explode_op: SQLGlotExpression = Unnest( - expressions=[explode_expr], - offset=sqlglot_expressions.Literal.number(1), - alias=TableAlias( - this=Identifier(this=lateral_alias), - columns=[ - sqlglot_expressions.Identifier(this="val"), - sqlglot_expressions.Identifier(this="idx"), - ], - ), - ) + explode_op: SQLGlotExpression + if val_index is None: + explode_op = Unnest( + expressions=[explode_expr], + alias=TableAlias( + this=Identifier(this=lateral_alias), + columns=[ + sqlglot_expressions.Identifier(this="val"), + ], + ), + ) + else: + explode_op = Unnest( + expressions=[explode_expr], + offset=sqlglot_expressions.Literal.number(1), + alias=TableAlias( + this=Identifier(this=lateral_alias), + columns=[ + sqlglot_expressions.Identifier(this="val"), + sqlglot_expressions.Identifier(this="idx"), + ], + ), + ) result = ( Select() .select(*column_exprs) .from_(Subquery(this=input_expr)) - .join( - Lateral( - this=explode_op, - ) - ) + .join(Lateral(this=explode_op)) ) return result diff --git a/tests/test_sql_refsols/explode_01_databricks.sql b/tests/test_sql_refsols/explode_01_databricks.sql new file mode 100644 index 000000000..1b56dd853 --- /dev/null +++ b/tests/test_sql_refsols/explode_01_databricks.sql @@ -0,0 +1,20 @@ +WITH _s1 AS ( + SELECT + n_regionkey AS region_key, + COLLECT_LIST(n_name) AS nation_names + FROM tpch.nation + GROUP BY + 1 +) +SELECT + region.r_name AS region_name, + _s1.nation_names, + _s2.idx AS nation_idx, + _s2.val AS nation_name +FROM tpch.region AS region +JOIN _s1 AS _s1 + ON _s1.region_key = region.r_regionkey +CROSS JOIN LATERAL POSEXPLODE(_s1.nation_names) AS _s2(idx, val) +ORDER BY + 1, + 3 diff --git a/tests/test_sql_refsols/explode_02_databricks.sql b/tests/test_sql_refsols/explode_02_databricks.sql new file mode 100644 index 000000000..4ec79c572 --- /dev/null +++ b/tests/test_sql_refsols/explode_02_databricks.sql @@ -0,0 +1,14 @@ +SELECT + tbl.key, + tbl.arr, + _s0.idx AS arr_idx, + _s0.val AS arr_val +FROM VALUES + ('A', ARRAY(1)), + ('B', ARRAY()), + ('C', ARRAY(2, 3, NULL, 4)), + ('D', ARRAY(5, 6)) AS tbl(key, arr) +CROSS JOIN LATERAL POSEXPLODE(tbl.arr) AS _s0(idx, val) +ORDER BY + 1, + 3 diff --git a/tests/test_sql_refsols/explode_03_databricks.sql b/tests/test_sql_refsols/explode_03_databricks.sql new file mode 100644 index 000000000..4f533f7bd --- /dev/null +++ b/tests/test_sql_refsols/explode_03_databricks.sql @@ -0,0 +1,16 @@ +WITH _q_0 AS ( + SELECT + c_name AS name + FROM tpch.customer + ORDER BY + c_custkey + LIMIT 5 +) +SELECT + _s0.val, + _s0.idx +FROM _q_0 AS _q_0 +CROSS JOIN LATERAL POSEXPLODE(SPLIT(_q_0.name, '\\Q#\\E')) AS _s0(idx, val) +ORDER BY + _q_0.name, + 2 diff --git a/tests/test_sql_refsols/explode_04_databricks.sql b/tests/test_sql_refsols/explode_04_databricks.sql new file mode 100644 index 000000000..0d2069f02 --- /dev/null +++ b/tests/test_sql_refsols/explode_04_databricks.sql @@ -0,0 +1,44 @@ +WITH _q_0 AS ( + SELECT + r_regionkey AS key, + r_name AS name + FROM tpch.region +), _s2 AS ( + SELECT + _q_0.key, + COUNT(*) AS n_rows + FROM _q_0 AS _q_0 + CROSS JOIN LATERAL POSEXPLODE(SPLIT(_q_0.name, '\\QE\\E')) AS _s0(idx, val) + GROUP BY + 1 +), _s5 AS ( + SELECT + _q_1.key, + COUNT(*) AS n_rows + FROM _q_0 AS _q_1 + CROSS JOIN LATERAL POSEXPLODE(SPLIT(_q_1.name, '\\QI\\E')) AS _s3(idx, val) + GROUP BY + 1 +), _s8 AS ( + SELECT + _q_2.key, + COUNT(*) AS n_rows + FROM _q_0 AS _q_2 + CROSS JOIN LATERAL POSEXPLODE(SPLIT(_q_2.name, '\\Q \\E')) AS _s6(idx, val) + GROUP BY + 1 +) +SELECT + region.r_name AS region_name, + COALESCE(_s2.n_rows, 0) AS n_e_chunks, + COALESCE(_s5.n_rows, 0) AS n_i_chunks, + COALESCE(_s8.n_rows, 0) AS n_space_chunks +FROM tpch.region AS region +LEFT JOIN _s2 AS _s2 + ON _s2.key = region.r_regionkey +LEFT JOIN _s5 AS _s5 + ON _s5.key = region.r_regionkey +LEFT JOIN _s8 AS _s8 + ON _s8.key = region.r_regionkey +ORDER BY + 1 diff --git a/tests/test_sql_refsols/explode_07_databricks.sql b/tests/test_sql_refsols/explode_07_databricks.sql new file mode 100644 index 000000000..23dc9ccd6 --- /dev/null +++ b/tests/test_sql_refsols/explode_07_databricks.sql @@ -0,0 +1,21 @@ +WITH _q_0 AS ( + SELECT + c_comment AS comment, + c_custkey AS key + FROM tpch.customer + ORDER BY + 2 + LIMIT 3 +) +SELECT + _q_0.key, + _s0.idx AS idx1, + _s1.idx AS idx2, + _s1.val AS val2 +FROM _q_0 AS _q_0 +CROSS JOIN LATERAL POSEXPLODE(SPLIT(_q_0.comment, '\\Q.\\E')) AS _s0(idx, val) +CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s0.val, '\\Q,\\E')) AS _s1(idx, val) +ORDER BY + 1, + 2, + 3 diff --git a/tests/test_sql_refsols/explode_08_databricks.sql b/tests/test_sql_refsols/explode_08_databricks.sql new file mode 100644 index 000000000..ef24083d3 --- /dev/null +++ b/tests/test_sql_refsols/explode_08_databricks.sql @@ -0,0 +1,25 @@ +WITH _q_0 AS ( + SELECT + c_comment AS comment, + c_custkey AS key + FROM tpch.customer + ORDER BY + 2 + LIMIT 3 +) +SELECT + _q_0.key, + _s0.idx AS idx1, + _s1.idx AS idx2, + _s2.idx AS idx3, + _s2.val AS val3 +FROM _q_0 AS _q_0 +CROSS JOIN LATERAL POSEXPLODE(SPLIT(_q_0.comment, '\\Q.\\E')) AS _s0(idx, val) +CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s0.val, '\\Q \\E')) AS _s1(idx, val) +JOIN LATERAL POSEXPLODE(SPLIT(_s1.val, '\\Q,\\E')) AS _s2(idx, val) + ON _s2.val <> '' +ORDER BY + 1, + 2, + 3, + 4 diff --git a/tests/test_sql_refsols/explode_09_databricks.sql b/tests/test_sql_refsols/explode_09_databricks.sql new file mode 100644 index 000000000..4afcf4993 --- /dev/null +++ b/tests/test_sql_refsols/explode_09_databricks.sql @@ -0,0 +1,10 @@ +SELECT + region.r_name AS name, + COUNT(*) AS n_words, + COLLECT_LIST(_s0.val) AS words_list +FROM tpch.region AS region +CROSS JOIN LATERAL POSEXPLODE(SPLIT(region.r_comment, '\\Q \\E')) AS _s0(idx, val) +GROUP BY + 1 +ORDER BY + 1 diff --git a/tests/test_sql_refsols/explode_10_databricks.sql b/tests/test_sql_refsols/explode_10_databricks.sql new file mode 100644 index 000000000..90158d90d --- /dev/null +++ b/tests/test_sql_refsols/explode_10_databricks.sql @@ -0,0 +1,15 @@ +SELECT + LENGTH(_s0.val) AS word_length, + COUNT(*) AS n_words, + COUNT(DISTINCT _s0.val) AS n_unique_words +FROM tpch.customer AS customer +CROSS JOIN LATERAL POSEXPLODE( + SPLIT( + TRIM(' ' FROM REPLACE(REPLACE(REPLACE(REPLACE(customer.c_comment, ';', ''), ',', ''), ':', ''), '.', '')), + '\\Q \\E' + ) +) AS _s0(idx, val) +GROUP BY + 1 +ORDER BY + 1 diff --git a/tests/test_sql_refsols/explode_11_databricks.sql b/tests/test_sql_refsols/explode_11_databricks.sql new file mode 100644 index 000000000..7293b56b5 --- /dev/null +++ b/tests/test_sql_refsols/explode_11_databricks.sql @@ -0,0 +1,30 @@ +WITH _s3 AS ( + SELECT DISTINCT + _s1.val AS cust_word + FROM tpch.customer AS customer + CROSS JOIN LATERAL POSEXPLODE( + SPLIT( + TRIM(' ' FROM REPLACE(REPLACE(REPLACE(REPLACE(customer.c_comment, ';', ''), ',', ''), ':', ''), '.', '')), + '\\Q \\E' + ) + ) AS _s1(idx, val) +), _u_0 AS ( + SELECT + cust_word AS _u_1 + FROM _s3 + GROUP BY + 1 +) +SELECT + COUNT(DISTINCT _s0.val) AS n_double_words +FROM tpch.supplier AS supplier +CROSS JOIN LATERAL POSEXPLODE( + SPLIT( + TRIM(' ' FROM REPLACE(REPLACE(REPLACE(REPLACE(supplier.s_comment, ';', ''), ',', ''), ':', ''), '.', '')), + '\\Q \\E' + ) +) AS _s0(idx, val) +LEFT JOIN _u_0 AS _u_0 + ON _s0.val = _u_0._u_1 +WHERE + NOT _u_0._u_1 IS NULL diff --git a/tests/test_sql_refsols/explode_12_databricks.sql b/tests/test_sql_refsols/explode_12_databricks.sql new file mode 100644 index 000000000..d847e4264 --- /dev/null +++ b/tests/test_sql_refsols/explode_12_databricks.sql @@ -0,0 +1,9 @@ +SELECT + tbl.key, + _s0.val AS arr_val +FROM VALUES + ('A', ARRAY(1)), + ('B', ARRAY()), + ('C', ARRAY(2, 3, NULL, 4)), + ('D', ARRAY(5, 6)) AS tbl(key, arr) +CROSS JOIN LATERAL POSEXPLODE(tbl.arr) AS _s0(idx, val) From f3752312467bc862d5974a288f3faf65270eb1c4 Mon Sep 17 00:00:00 2001 From: knassre-bodo Date: Wed, 12 Aug 2026 10:58:42 -0700 Subject: [PATCH 27/44] Added duckdb explode support --- .../duckdb_transform_bindings.py | 80 +++++++++++++++++++ tests/test_pipeline_tpch_custom.py | 1 + tests/test_sql_refsols/explode_01_duckdb.sql | 24 ++++++ tests/test_sql_refsols/explode_02_duckdb.sql | 18 +++++ tests/test_sql_refsols/explode_03_duckdb.sql | 20 +++++ tests/test_sql_refsols/explode_04_duckdb.sql | 56 +++++++++++++ tests/test_sql_refsols/explode_07_duckdb.sql | 29 +++++++ tests/test_sql_refsols/explode_08_duckdb.sql | 37 +++++++++ tests/test_sql_refsols/explode_09_duckdb.sql | 14 ++++ tests/test_sql_refsols/explode_10_duckdb.sql | 31 +++++++ tests/test_sql_refsols/explode_11_duckdb.sql | 62 ++++++++++++++ tests/test_sql_refsols/explode_12_duckdb.sql | 13 +++ 12 files changed, 385 insertions(+) create mode 100644 tests/test_sql_refsols/explode_01_duckdb.sql create mode 100644 tests/test_sql_refsols/explode_02_duckdb.sql create mode 100644 tests/test_sql_refsols/explode_03_duckdb.sql create mode 100644 tests/test_sql_refsols/explode_04_duckdb.sql create mode 100644 tests/test_sql_refsols/explode_07_duckdb.sql create mode 100644 tests/test_sql_refsols/explode_08_duckdb.sql create mode 100644 tests/test_sql_refsols/explode_09_duckdb.sql create mode 100644 tests/test_sql_refsols/explode_10_duckdb.sql create mode 100644 tests/test_sql_refsols/explode_11_duckdb.sql create mode 100644 tests/test_sql_refsols/explode_12_duckdb.sql diff --git a/pydough/sqlglot/transform_bindings/duckdb_transform_bindings.py b/pydough/sqlglot/transform_bindings/duckdb_transform_bindings.py index da28e8d91..5915e42d9 100644 --- a/pydough/sqlglot/transform_bindings/duckdb_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/duckdb_transform_bindings.py @@ -9,8 +9,16 @@ import sqlglot.expressions as sqlglot_expressions from sqlglot.expressions import Expression as SQLGlotExpression +from sqlglot.expressions import ( + Identifier, + Lateral, + Select, + Subquery, + TableAlias, +) from pydough.types import NumericType, PyDoughType, StringType +from pydough.utilities import ExplodeSpec from .base_transform_bindings import BaseTransformBindings from .sqlglot_transform_utils import DateTimeUnit, apply_parens @@ -130,6 +138,78 @@ def convert_listof( ) -> SQLGlotExpression: return sqlglot_expressions.ArrayAgg(this=args[0]) + def convert_explode( + self, + input_expr: SQLGlotExpression, + explode_expr: SQLGlotExpression, + explode_spec: ExplodeSpec, + exprs: list[SQLGlotExpression], + val_index: int | None, + idx_index: int | None, + lateral_alias: str, + ) -> SQLGlotExpression: + column_exprs: list[SQLGlotExpression] = [*exprs] + if val_index is not None: + column_exprs[val_index] = sqlglot_expressions.Alias( + this=sqlglot_expressions.Column( + this=sqlglot_expressions.Identifier(this="val"), + table=sqlglot_expressions.Identifier(this=lateral_alias), + ), + alias=sqlglot_expressions.Identifier(this=explode_spec.value_name), + ) + if idx_index is not None and explode_spec.index_name is not None: + column_exprs[idx_index] = sqlglot_expressions.Alias( + this=sqlglot_expressions.Column( + this=sqlglot_expressions.Identifier(this="idx"), + table=sqlglot_expressions.Identifier(this=lateral_alias), + ), + alias=sqlglot_expressions.Identifier(this=explode_spec.index_name), + ) + + if explode_spec.version == "string": + assert explode_spec.delimiter is not None, ( + "Delimiter must be provided for string explode." + ) + explode_expr = sqlglot_expressions.Split( + this=explode_expr, + expression=sqlglot_expressions.Literal.string(explode_spec.delimiter), + ) + explode_args: list[SQLGlotExpression] = [ + sqlglot_expressions.Unnest(expressions=[explode_expr]) + ] + lateral_columns: list[SQLGlotExpression] = [ + sqlglot_expressions.Identifier(this="val") + ] + if val_index is not None: + explode_args.append( + sqlglot_expressions.Sub( + this=sqlglot_expressions.Anonymous( + this="generate_subscripts", + expressions=[ + explode_expr, + sqlglot_expressions.Literal.number(1), + ], + ), + expression=sqlglot_expressions.Literal.number(1), + ) + ) + lateral_columns.append(sqlglot_expressions.Identifier(this="idx")) + result = ( + Select() + .select(*column_exprs) + .from_(Subquery(this=input_expr)) + .join( + Lateral( + this=Subquery(this=Select().select(*explode_args)), + alias=TableAlias( + this=Identifier(this=lateral_alias), columns=lateral_columns + ), + ) + ) + ) + + return result + def generate_dataframe_item_dialect_expression( self, item: Any, item_type: PyDoughType ) -> SQLGlotExpression: diff --git a/tests/test_pipeline_tpch_custom.py b/tests/test_pipeline_tpch_custom.py index 080e85fb6..bd617d2cf 100644 --- a/tests/test_pipeline_tpch_custom.py +++ b/tests/test_pipeline_tpch_custom.py @@ -4734,6 +4734,7 @@ ), "explode_09", order_sensitive=True, + ignore_array_order=True, skipped_dialects={"ANSI", "SQLITE", "MYSQL"}, ), id="explode_09", diff --git a/tests/test_sql_refsols/explode_01_duckdb.sql b/tests/test_sql_refsols/explode_01_duckdb.sql new file mode 100644 index 000000000..570bcb4ed --- /dev/null +++ b/tests/test_sql_refsols/explode_01_duckdb.sql @@ -0,0 +1,24 @@ +WITH _s1 AS ( + SELECT + n_regionkey AS region_key, + ARRAY_AGG(n_name) AS nation_names + FROM tpch.nation + GROUP BY + 1 +) +SELECT + region.r_name AS region_name, + _s1.nation_names, + _s2.idx AS nation_idx, + _s2.val AS nation_name +FROM tpch.region AS region +JOIN _s1 AS _s1 + ON _s1.region_key = region.r_regionkey +CROSS JOIN LATERAL ( + SELECT + UNNEST(_s1.nation_names) AS _col_0, + GENERATE_SUBSCRIPTS(_s1.nation_names, 1) - 1 AS _col_1 +) AS _s2(val, idx) +ORDER BY + 1 NULLS FIRST, + 3 NULLS FIRST diff --git a/tests/test_sql_refsols/explode_02_duckdb.sql b/tests/test_sql_refsols/explode_02_duckdb.sql new file mode 100644 index 000000000..ea1190dce --- /dev/null +++ b/tests/test_sql_refsols/explode_02_duckdb.sql @@ -0,0 +1,18 @@ +SELECT + tbl.key, + tbl.arr, + _s0.idx AS arr_idx, + _s0.val AS arr_val +FROM (VALUES + ('A', [1]), + ('B', []), + ('C', [2, 3, NULL, 4]), + ('D', [5, 6])) AS tbl(key, arr) +CROSS JOIN LATERAL ( + SELECT + UNNEST(tbl.arr) AS _col_0, + GENERATE_SUBSCRIPTS(tbl.arr, 1) - 1 AS _col_1 +) AS _s0(val, idx) +ORDER BY + 1 NULLS FIRST, + 3 NULLS FIRST diff --git a/tests/test_sql_refsols/explode_03_duckdb.sql b/tests/test_sql_refsols/explode_03_duckdb.sql new file mode 100644 index 000000000..cd0c9fc54 --- /dev/null +++ b/tests/test_sql_refsols/explode_03_duckdb.sql @@ -0,0 +1,20 @@ +WITH _q_0 AS ( + SELECT + c_name AS name + FROM tpch.customer + ORDER BY + c_custkey NULLS FIRST + LIMIT 5 +) +SELECT + _s0.val, + _s0.idx +FROM _q_0 AS _q_0 +CROSS JOIN LATERAL ( + SELECT + UNNEST(STR_SPLIT(_q_0.name, '#')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_q_0.name, '#'), 1) - 1 AS _col_1 +) AS _s0(val, idx) +ORDER BY + _q_0.name NULLS FIRST, + 2 NULLS FIRST diff --git a/tests/test_sql_refsols/explode_04_duckdb.sql b/tests/test_sql_refsols/explode_04_duckdb.sql new file mode 100644 index 000000000..392f55815 --- /dev/null +++ b/tests/test_sql_refsols/explode_04_duckdb.sql @@ -0,0 +1,56 @@ +WITH _q_0 AS ( + SELECT + r_regionkey AS key, + r_name AS name + FROM tpch.region +), _s2 AS ( + SELECT + _q_0.key, + COUNT(*) AS n_rows + FROM _q_0 AS _q_0 + CROSS JOIN LATERAL ( + SELECT + UNNEST(STR_SPLIT(_q_0.name, 'E')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_q_0.name, 'E'), 1) - 1 AS _col_1 + ) AS _s0(val, idx) + GROUP BY + 1 +), _s5 AS ( + SELECT + _q_1.key, + COUNT(*) AS n_rows + FROM _q_0 AS _q_1 + CROSS JOIN LATERAL ( + SELECT + UNNEST(STR_SPLIT(_q_1.name, 'I')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_q_1.name, 'I'), 1) - 1 AS _col_1 + ) AS _s3(val, idx) + GROUP BY + 1 +), _s8 AS ( + SELECT + _q_2.key, + COUNT(*) AS n_rows + FROM _q_0 AS _q_2 + CROSS JOIN LATERAL ( + SELECT + UNNEST(STR_SPLIT(_q_2.name, ' ')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_q_2.name, ' '), 1) - 1 AS _col_1 + ) AS _s6(val, idx) + GROUP BY + 1 +) +SELECT + region.r_name AS region_name, + COALESCE(_s2.n_rows, 0) AS n_e_chunks, + COALESCE(_s5.n_rows, 0) AS n_i_chunks, + COALESCE(_s8.n_rows, 0) AS n_space_chunks +FROM tpch.region AS region +LEFT JOIN _s2 AS _s2 + ON _s2.key = region.r_regionkey +LEFT JOIN _s5 AS _s5 + ON _s5.key = region.r_regionkey +LEFT JOIN _s8 AS _s8 + ON _s8.key = region.r_regionkey +ORDER BY + 1 NULLS FIRST diff --git a/tests/test_sql_refsols/explode_07_duckdb.sql b/tests/test_sql_refsols/explode_07_duckdb.sql new file mode 100644 index 000000000..3457e18b1 --- /dev/null +++ b/tests/test_sql_refsols/explode_07_duckdb.sql @@ -0,0 +1,29 @@ +WITH _q_0 AS ( + SELECT + c_comment AS comment, + c_custkey AS key + FROM tpch.customer + ORDER BY + 2 NULLS FIRST + LIMIT 3 +) +SELECT + _q_0.key, + _s0.idx AS idx1, + _s1.idx AS idx2, + _s1.val AS val2 +FROM _q_0 AS _q_0 +CROSS JOIN LATERAL ( + SELECT + UNNEST(STR_SPLIT(_q_0.comment, '.')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_q_0.comment, '.'), 1) - 1 AS _col_1 +) AS _s0(val, idx) +CROSS JOIN LATERAL ( + SELECT + UNNEST(STR_SPLIT(_s0.val, ',')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_s0.val, ','), 1) - 1 AS _col_1 +) AS _s1(val, idx) +ORDER BY + 1 NULLS FIRST, + 2 NULLS FIRST, + 3 NULLS FIRST diff --git a/tests/test_sql_refsols/explode_08_duckdb.sql b/tests/test_sql_refsols/explode_08_duckdb.sql new file mode 100644 index 000000000..05c702f39 --- /dev/null +++ b/tests/test_sql_refsols/explode_08_duckdb.sql @@ -0,0 +1,37 @@ +WITH _q_0 AS ( + SELECT + c_comment AS comment, + c_custkey AS key + FROM tpch.customer + ORDER BY + 2 NULLS FIRST + LIMIT 3 +) +SELECT + _q_0.key, + _s0.idx AS idx1, + _s1.idx AS idx2, + _s2.idx AS idx3, + _s2.val AS val3 +FROM _q_0 AS _q_0 +CROSS JOIN LATERAL ( + SELECT + UNNEST(STR_SPLIT(_q_0.comment, '.')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_q_0.comment, '.'), 1) - 1 AS _col_1 +) AS _s0(val, idx) +CROSS JOIN LATERAL ( + SELECT + UNNEST(STR_SPLIT(_s0.val, ' ')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_s0.val, ' '), 1) - 1 AS _col_1 +) AS _s1(val, idx) +JOIN LATERAL ( + SELECT + UNNEST(STR_SPLIT(_s1.val, ',')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_s1.val, ','), 1) - 1 AS _col_1 +) AS _s2(val, idx) + ON _s2.val <> '' +ORDER BY + 1 NULLS FIRST, + 2 NULLS FIRST, + 3 NULLS FIRST, + 4 NULLS FIRST diff --git a/tests/test_sql_refsols/explode_09_duckdb.sql b/tests/test_sql_refsols/explode_09_duckdb.sql new file mode 100644 index 000000000..8283172ac --- /dev/null +++ b/tests/test_sql_refsols/explode_09_duckdb.sql @@ -0,0 +1,14 @@ +SELECT + region.r_name AS name, + COUNT(*) AS n_words, + ARRAY_AGG(_s0.val) AS words_list +FROM tpch.region AS region +CROSS JOIN LATERAL ( + SELECT + UNNEST(STR_SPLIT(region.r_comment, ' ')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(region.r_comment, ' '), 1) - 1 AS _col_1 +) AS _s0(val, idx) +GROUP BY + 1 +ORDER BY + 1 NULLS FIRST diff --git a/tests/test_sql_refsols/explode_10_duckdb.sql b/tests/test_sql_refsols/explode_10_duckdb.sql new file mode 100644 index 000000000..63a3b3029 --- /dev/null +++ b/tests/test_sql_refsols/explode_10_duckdb.sql @@ -0,0 +1,31 @@ +SELECT + LENGTH(_s0.val) AS word_length, + COUNT(*) AS n_words, + COUNT(DISTINCT _s0.val) AS n_unique_words +FROM tpch.customer AS customer +CROSS JOIN LATERAL ( + SELECT + UNNEST( + STR_SPLIT( + TRIM( + REPLACE(REPLACE(REPLACE(REPLACE(customer.c_comment, ';', ''), ',', ''), ':', ''), '.', ''), + ' ' + ), + ' ' + ) + ) AS _col_0, + GENERATE_SUBSCRIPTS( + STR_SPLIT( + TRIM( + REPLACE(REPLACE(REPLACE(REPLACE(customer.c_comment, ';', ''), ',', ''), ':', ''), '.', ''), + ' ' + ), + ' ' + ), + 1 + ) - 1 AS _col_1 +) AS _s0(val, idx) +GROUP BY + 1 +ORDER BY + 1 NULLS FIRST diff --git a/tests/test_sql_refsols/explode_11_duckdb.sql b/tests/test_sql_refsols/explode_11_duckdb.sql new file mode 100644 index 000000000..75d8e521b --- /dev/null +++ b/tests/test_sql_refsols/explode_11_duckdb.sql @@ -0,0 +1,62 @@ +WITH _s3 AS ( + SELECT DISTINCT + _s1.val AS cust_word + FROM tpch.customer AS customer + CROSS JOIN LATERAL ( + SELECT + UNNEST( + STR_SPLIT( + TRIM( + REPLACE(REPLACE(REPLACE(REPLACE(customer.c_comment, ';', ''), ',', ''), ':', ''), '.', ''), + ' ' + ), + ' ' + ) + ) AS _col_0, + GENERATE_SUBSCRIPTS( + STR_SPLIT( + TRIM( + REPLACE(REPLACE(REPLACE(REPLACE(customer.c_comment, ';', ''), ',', ''), ':', ''), '.', ''), + ' ' + ), + ' ' + ), + 1 + ) - 1 AS _col_1 + ) AS _s1(val, idx) +), _u_0 AS ( + SELECT + cust_word AS _u_1 + FROM _s3 + GROUP BY + 1 +) +SELECT + COUNT(DISTINCT _s0.val) AS n_double_words +FROM tpch.supplier AS supplier +CROSS JOIN LATERAL ( + SELECT + UNNEST( + STR_SPLIT( + TRIM( + REPLACE(REPLACE(REPLACE(REPLACE(supplier.s_comment, ';', ''), ',', ''), ':', ''), '.', ''), + ' ' + ), + ' ' + ) + ) AS _col_0, + GENERATE_SUBSCRIPTS( + STR_SPLIT( + TRIM( + REPLACE(REPLACE(REPLACE(REPLACE(supplier.s_comment, ';', ''), ',', ''), ':', ''), '.', ''), + ' ' + ), + ' ' + ), + 1 + ) - 1 AS _col_1 +) AS _s0(val, idx) +LEFT JOIN _u_0 AS _u_0 + ON _s0.val = _u_0._u_1 +WHERE + NOT _u_0._u_1 IS NULL diff --git a/tests/test_sql_refsols/explode_12_duckdb.sql b/tests/test_sql_refsols/explode_12_duckdb.sql new file mode 100644 index 000000000..fc6df41e2 --- /dev/null +++ b/tests/test_sql_refsols/explode_12_duckdb.sql @@ -0,0 +1,13 @@ +SELECT + tbl.key, + _s0.val AS arr_val +FROM (VALUES + ('A', [1]), + ('B', []), + ('C', [2, 3, NULL, 4]), + ('D', [5, 6])) AS tbl(key, arr) +CROSS JOIN LATERAL ( + SELECT + UNNEST(tbl.arr) AS _col_0, + GENERATE_SUBSCRIPTS(tbl.arr, 1) - 1 AS _col_1 +) AS _s0(val, idx) From 24de8648a51d6c17ce281f5b8e23835d20bdfd6b Mon Sep 17 00:00:00 2001 From: knassre-bodo Date: Fri, 14 Aug 2026 10:04:32 -0700 Subject: [PATCH 28/44] Adding trino support --- .../trino_transform_bindings.py | 76 +++++++++++++++++++ tests/test_sql_refsols/explode_01_trino.sql | 20 +++++ tests/test_sql_refsols/explode_02_trino.sql | 14 ++++ tests/test_sql_refsols/explode_03_trino.sql | 16 ++++ tests/test_sql_refsols/explode_04_trino.sql | 44 +++++++++++ tests/test_sql_refsols/explode_07_trino.sql | 21 +++++ tests/test_sql_refsols/explode_08_trino.sql | 25 ++++++ tests/test_sql_refsols/explode_09_trino.sql | 10 +++ tests/test_sql_refsols/explode_10_trino.sql | 13 ++++ tests/test_sql_refsols/explode_11_trino.sql | 26 +++++++ tests/test_sql_refsols/explode_12_trino.sql | 9 +++ 11 files changed, 274 insertions(+) create mode 100644 tests/test_sql_refsols/explode_01_trino.sql create mode 100644 tests/test_sql_refsols/explode_02_trino.sql create mode 100644 tests/test_sql_refsols/explode_03_trino.sql create mode 100644 tests/test_sql_refsols/explode_04_trino.sql create mode 100644 tests/test_sql_refsols/explode_07_trino.sql create mode 100644 tests/test_sql_refsols/explode_08_trino.sql create mode 100644 tests/test_sql_refsols/explode_09_trino.sql create mode 100644 tests/test_sql_refsols/explode_10_trino.sql create mode 100644 tests/test_sql_refsols/explode_11_trino.sql create mode 100644 tests/test_sql_refsols/explode_12_trino.sql diff --git a/pydough/sqlglot/transform_bindings/trino_transform_bindings.py b/pydough/sqlglot/transform_bindings/trino_transform_bindings.py index 98a73c283..cf21dfbaa 100644 --- a/pydough/sqlglot/transform_bindings/trino_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/trino_transform_bindings.py @@ -10,6 +10,13 @@ import sqlglot.expressions as sqlglot_expressions from sqlglot.expressions import Expression as SQLGlotExpression +from sqlglot.expressions import ( + Identifier, + Select, + Subquery, + TableAlias, + Unnest, +) import pydough.pydough_operators as pydop from pydough.configs import DayOfWeek @@ -18,6 +25,7 @@ PyDoughType, StringType, ) +from pydough.utilities import ExplodeSpec from .base_transform_bindings import BaseTransformBindings from .sqlglot_transform_utils import DateTimeUnit, apply_parens @@ -116,6 +124,74 @@ def generate_dataframe_array_expression( ) -> SQLGlotExpression: return sqlglot_expressions.Array(expressions=items) + def convert_explode( + self, + input_expr: SQLGlotExpression, + explode_expr: SQLGlotExpression, + explode_spec: ExplodeSpec, + exprs: list[SQLGlotExpression], + val_index: int | None, + idx_index: int | None, + lateral_alias: str, + ) -> SQLGlotExpression: + column_exprs: list[SQLGlotExpression] = [*exprs] + if val_index is not None: + column_exprs[val_index] = sqlglot_expressions.Alias( + this=sqlglot_expressions.Column( + this=sqlglot_expressions.Identifier(this="val"), + table=sqlglot_expressions.Identifier(this=lateral_alias), + ), + alias=sqlglot_expressions.Identifier(this=explode_spec.value_name), + ) + if idx_index is not None and explode_spec.index_name is not None: + column_exprs[idx_index] = sqlglot_expressions.Alias( + this=sqlglot_expressions.Sub( + this=sqlglot_expressions.Column( + this=sqlglot_expressions.Identifier(this="idx"), + table=sqlglot_expressions.Identifier(this=lateral_alias), + ), + expression=sqlglot_expressions.Literal.number(1), + ), + alias=sqlglot_expressions.Identifier(this=explode_spec.index_name), + ) + + if explode_spec.version == "string": + assert explode_spec.delimiter is not None, ( + "Delimiter must be provided for string explode." + ) + explode_expr = sqlglot_expressions.Split( + this=explode_expr, + expression=sqlglot_expressions.Literal.string(explode_spec.delimiter), + ) + explode_op: SQLGlotExpression + if val_index is None: + explode_op = Unnest( + expressions=[explode_expr], + alias=TableAlias( + this=Identifier(this=lateral_alias), + columns=[ + sqlglot_expressions.Identifier(this="val"), + ], + ), + ) + else: + explode_op = Unnest( + expressions=[explode_expr], + alias=TableAlias( + this=Identifier(this=lateral_alias), + columns=[sqlglot_expressions.Identifier(this="val")], + ), + offset=sqlglot_expressions.Identifier(this="idx"), + ) + result = ( + Select() + .select(*column_exprs) + .from_(Subquery(this=input_expr)) + .join(explode_op) + ) + + return result + def convert_datediff( self, args: list[SQLGlotExpression], diff --git a/tests/test_sql_refsols/explode_01_trino.sql b/tests/test_sql_refsols/explode_01_trino.sql new file mode 100644 index 000000000..0836d4cc6 --- /dev/null +++ b/tests/test_sql_refsols/explode_01_trino.sql @@ -0,0 +1,20 @@ +WITH _s1 AS ( + SELECT + n_regionkey AS region_key, + ARRAY_AGG(n_name) AS nation_names + FROM tpch.nation + GROUP BY + 1 +) +SELECT + region.r_name AS region_name, + _s1.nation_names, + _s2.idx - 1 AS nation_idx, + _s2.val AS nation_name +FROM tpch.region AS region +JOIN _s1 AS _s1 + ON _s1.region_key = region.r_regionkey +CROSS JOIN UNNEST(_s1.nation_names) WITH ORDINALITY AS _s2(val, idx) +ORDER BY + 1 NULLS FIRST, + 3 NULLS FIRST diff --git a/tests/test_sql_refsols/explode_02_trino.sql b/tests/test_sql_refsols/explode_02_trino.sql new file mode 100644 index 000000000..fae563687 --- /dev/null +++ b/tests/test_sql_refsols/explode_02_trino.sql @@ -0,0 +1,14 @@ +SELECT + tbl.key, + tbl.arr, + _s0.idx - 1 AS arr_idx, + _s0.val AS arr_val +FROM (VALUES + ('A', ARRAY[1]), + ('B', ARRAY[]), + ('C', ARRAY[2, 3, NULL, 4]), + ('D', ARRAY[5, 6])) AS tbl(key, arr) +CROSS JOIN UNNEST(tbl.arr) WITH ORDINALITY AS _s0(val, idx) +ORDER BY + 1 NULLS FIRST, + 3 NULLS FIRST diff --git a/tests/test_sql_refsols/explode_03_trino.sql b/tests/test_sql_refsols/explode_03_trino.sql new file mode 100644 index 000000000..e6674d08a --- /dev/null +++ b/tests/test_sql_refsols/explode_03_trino.sql @@ -0,0 +1,16 @@ +WITH _q_0 AS ( + SELECT + c_name AS name + FROM tpch.customer + ORDER BY + c_custkey NULLS FIRST + LIMIT 5 +) +SELECT + _s0.val, + _s0.idx - 1 AS idx +FROM _q_0 AS _q_0 +CROSS JOIN UNNEST(SPLIT(_q_0.name, '#')) WITH ORDINALITY AS _s0(val, idx) +ORDER BY + _q_0.name NULLS FIRST, + 2 NULLS FIRST diff --git a/tests/test_sql_refsols/explode_04_trino.sql b/tests/test_sql_refsols/explode_04_trino.sql new file mode 100644 index 000000000..a631c8fb3 --- /dev/null +++ b/tests/test_sql_refsols/explode_04_trino.sql @@ -0,0 +1,44 @@ +WITH _q_0 AS ( + SELECT + r_regionkey AS key, + r_name AS name + FROM tpch.region +), _s2 AS ( + SELECT + _q_0.key, + COUNT(*) AS n_rows + FROM _q_0 AS _q_0 + CROSS JOIN UNNEST(SPLIT(_q_0.name, 'E')) WITH ORDINALITY AS _s0(val, idx) + GROUP BY + 1 +), _s5 AS ( + SELECT + _q_1.key, + COUNT(*) AS n_rows + FROM _q_0 AS _q_1 + CROSS JOIN UNNEST(SPLIT(_q_1.name, 'I')) WITH ORDINALITY AS _s3(val, idx) + GROUP BY + 1 +), _s8 AS ( + SELECT + _q_2.key, + COUNT(*) AS n_rows + FROM _q_0 AS _q_2 + CROSS JOIN UNNEST(SPLIT(_q_2.name, ' ')) WITH ORDINALITY AS _s6(val, idx) + GROUP BY + 1 +) +SELECT + region.r_name AS region_name, + COALESCE(_s2.n_rows, 0) AS n_e_chunks, + COALESCE(_s5.n_rows, 0) AS n_i_chunks, + COALESCE(_s8.n_rows, 0) AS n_space_chunks +FROM tpch.region AS region +LEFT JOIN _s2 AS _s2 + ON _s2.key = region.r_regionkey +LEFT JOIN _s5 AS _s5 + ON _s5.key = region.r_regionkey +LEFT JOIN _s8 AS _s8 + ON _s8.key = region.r_regionkey +ORDER BY + 1 NULLS FIRST diff --git a/tests/test_sql_refsols/explode_07_trino.sql b/tests/test_sql_refsols/explode_07_trino.sql new file mode 100644 index 000000000..888e37a3b --- /dev/null +++ b/tests/test_sql_refsols/explode_07_trino.sql @@ -0,0 +1,21 @@ +WITH _q_0 AS ( + SELECT + c_comment AS comment, + c_custkey AS key + FROM tpch.customer + ORDER BY + 2 NULLS FIRST + LIMIT 3 +) +SELECT + _q_0.key, + _s0.idx - 1 AS idx1, + _s1.idx - 1 AS idx2, + _s1.val AS val2 +FROM _q_0 AS _q_0 +CROSS JOIN UNNEST(SPLIT(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) +CROSS JOIN UNNEST(SPLIT(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx) +ORDER BY + 1 NULLS FIRST, + 2 NULLS FIRST, + 3 NULLS FIRST diff --git a/tests/test_sql_refsols/explode_08_trino.sql b/tests/test_sql_refsols/explode_08_trino.sql new file mode 100644 index 000000000..5cb8de532 --- /dev/null +++ b/tests/test_sql_refsols/explode_08_trino.sql @@ -0,0 +1,25 @@ +WITH _q_0 AS ( + SELECT + c_comment AS comment, + c_custkey AS key + FROM tpch.customer + ORDER BY + 2 NULLS FIRST + LIMIT 3 +) +SELECT + _q_0.key, + _s0.idx - 1 AS idx1, + _s1.idx - 1 AS idx2, + _s2.idx - 1 AS idx3, + _s2.val AS val3 +FROM _q_0 AS _q_0 +CROSS JOIN UNNEST(SPLIT(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) +CROSS JOIN UNNEST(SPLIT(_s0.val, ' ')) WITH ORDINALITY AS _s1(val, idx) +JOIN UNNEST(SPLIT(_s1.val, ',')) WITH ORDINALITY AS _s2(val, idx) + ON _s2.val <> '' +ORDER BY + 1 NULLS FIRST, + 2 NULLS FIRST, + 3 NULLS FIRST, + 4 NULLS FIRST diff --git a/tests/test_sql_refsols/explode_09_trino.sql b/tests/test_sql_refsols/explode_09_trino.sql new file mode 100644 index 000000000..3999d1f17 --- /dev/null +++ b/tests/test_sql_refsols/explode_09_trino.sql @@ -0,0 +1,10 @@ +SELECT + region.r_name AS name, + COUNT(*) AS n_words, + ARRAY_AGG(_s0.val) AS words_list +FROM tpch.region AS region +CROSS JOIN UNNEST(SPLIT(region.r_comment, ' ')) WITH ORDINALITY AS _s0(val, idx) +GROUP BY + 1 +ORDER BY + 1 NULLS FIRST diff --git a/tests/test_sql_refsols/explode_10_trino.sql b/tests/test_sql_refsols/explode_10_trino.sql new file mode 100644 index 000000000..d1eb3ab4f --- /dev/null +++ b/tests/test_sql_refsols/explode_10_trino.sql @@ -0,0 +1,13 @@ +SELECT + LENGTH(_s0.val) AS word_length, + COUNT(*) AS n_words, + COUNT(DISTINCT _s0.val) AS n_unique_words +FROM tpch.customer AS customer +CROSS JOIN UNNEST(SPLIT( + TRIM(' ' FROM REPLACE(REPLACE(REPLACE(REPLACE(customer.c_comment, ';', ''), ',', ''), ':', ''), '.', '')), + ' ' +)) WITH ORDINALITY AS _s0(val, idx) +GROUP BY + 1 +ORDER BY + 1 NULLS FIRST diff --git a/tests/test_sql_refsols/explode_11_trino.sql b/tests/test_sql_refsols/explode_11_trino.sql new file mode 100644 index 000000000..f9ef9f905 --- /dev/null +++ b/tests/test_sql_refsols/explode_11_trino.sql @@ -0,0 +1,26 @@ +WITH _s3 AS ( + SELECT DISTINCT + _s1.val AS cust_word + FROM tpch.customer AS customer + CROSS JOIN UNNEST(SPLIT( + TRIM(' ' FROM REPLACE(REPLACE(REPLACE(REPLACE(customer.c_comment, ';', ''), ',', ''), ':', ''), '.', '')), + ' ' + )) WITH ORDINALITY AS _s1(val, idx) +), _u_0 AS ( + SELECT + cust_word AS _u_1 + FROM _s3 + GROUP BY + 1 +) +SELECT + COUNT(DISTINCT _s0.val) AS n_double_words +FROM tpch.supplier AS supplier +CROSS JOIN UNNEST(SPLIT( + TRIM(' ' FROM REPLACE(REPLACE(REPLACE(REPLACE(supplier.s_comment, ';', ''), ',', ''), ':', ''), '.', '')), + ' ' +)) WITH ORDINALITY AS _s0(val, idx) +LEFT JOIN _u_0 AS _u_0 + ON _s0.val = _u_0._u_1 +WHERE + NOT _u_0._u_1 IS NULL diff --git a/tests/test_sql_refsols/explode_12_trino.sql b/tests/test_sql_refsols/explode_12_trino.sql new file mode 100644 index 000000000..1417404f4 --- /dev/null +++ b/tests/test_sql_refsols/explode_12_trino.sql @@ -0,0 +1,9 @@ +SELECT + tbl.key, + _s0.val AS arr_val +FROM (VALUES + ('A', ARRAY[1]), + ('B', ARRAY[]), + ('C', ARRAY[2, 3, NULL, 4]), + ('D', ARRAY[5, 6])) AS tbl(key, arr) +CROSS JOIN UNNEST(tbl.arr) WITH ORDINALITY AS _s0(val, idx) From 4a077e5dbe864125ef0953271726ef79df68d3df Mon Sep 17 00:00:00 2001 From: knassre-bodo Date: Fri, 14 Aug 2026 11:32:21 -0700 Subject: [PATCH 29/44] Added more tests, need to fix lateral ON condition [RUN DIALECTS] --- .../duckdb_transform_bindings.py | 19 +- .../postgres_transform_bindings.py | 23 +- .../sf_transform_bindings.py | 28 +- pydough/unqualified/unqualified_node.py | 8 +- tests/test_pipeline_tpch_custom.py | 509 +++++++++++++++++- tests/test_plan_refsols/explode_13.txt | 4 + tests/test_plan_refsols/explode_14.txt | 8 + tests/test_plan_refsols/explode_15.txt | 8 + tests/test_plan_refsols/explode_16.txt | 8 + tests/test_plan_refsols/explode_17.txt | 8 + tests/test_plan_refsols/explode_18.txt | 8 + tests/test_plan_refsols/explode_19.txt | 8 + tests/test_plan_refsols/explode_20.txt | 6 + tests/test_plan_refsols/explode_21.txt | 9 + tests/test_qualification_errors.py | 10 +- .../explode_13_databricks.sql | 9 + tests/test_sql_refsols/explode_13_duckdb.sql | 13 + .../test_sql_refsols/explode_13_postgres.sql | 11 + tests/test_sql_refsols/explode_13_trino.sql | 9 + .../explode_14_databricks.sql | 30 ++ tests/test_sql_refsols/explode_14_duckdb.sql | 42 ++ .../test_sql_refsols/explode_14_postgres.sql | 31 ++ .../test_sql_refsols/explode_14_snowflake.sql | 30 ++ tests/test_sql_refsols/explode_14_trino.sql | 31 ++ .../explode_15_databricks.sql | 30 ++ tests/test_sql_refsols/explode_15_duckdb.sql | 42 ++ .../test_sql_refsols/explode_15_postgres.sql | 31 ++ .../test_sql_refsols/explode_15_snowflake.sql | 30 ++ tests/test_sql_refsols/explode_15_trino.sql | 31 ++ .../explode_16_databricks.sql | 30 ++ tests/test_sql_refsols/explode_16_duckdb.sql | 42 ++ .../test_sql_refsols/explode_16_postgres.sql | 31 ++ .../test_sql_refsols/explode_16_snowflake.sql | 30 ++ tests/test_sql_refsols/explode_16_trino.sql | 31 ++ .../explode_17_databricks.sql | 30 ++ tests/test_sql_refsols/explode_17_duckdb.sql | 42 ++ .../test_sql_refsols/explode_17_postgres.sql | 31 ++ .../test_sql_refsols/explode_17_snowflake.sql | 30 ++ tests/test_sql_refsols/explode_17_trino.sql | 31 ++ .../explode_18_databricks.sql | 30 ++ tests/test_sql_refsols/explode_18_duckdb.sql | 42 ++ .../test_sql_refsols/explode_18_postgres.sql | 31 ++ .../test_sql_refsols/explode_18_snowflake.sql | 30 ++ tests/test_sql_refsols/explode_18_trino.sql | 31 ++ .../explode_19_databricks.sql | 30 ++ tests/test_sql_refsols/explode_19_duckdb.sql | 42 ++ .../test_sql_refsols/explode_19_postgres.sql | 31 ++ .../test_sql_refsols/explode_19_snowflake.sql | 30 ++ tests/test_sql_refsols/explode_19_trino.sql | 31 ++ .../explode_20_databricks.sql | 16 + tests/test_sql_refsols/explode_20_duckdb.sql | 20 + .../test_sql_refsols/explode_20_postgres.sql | 16 + .../test_sql_refsols/explode_20_snowflake.sql | 16 + tests/test_sql_refsols/explode_20_trino.sql | 16 + .../explode_21_databricks.sql | 33 ++ tests/test_sql_refsols/explode_21_duckdb.sql | 49 ++ .../test_sql_refsols/explode_21_postgres.sql | 34 ++ .../test_sql_refsols/explode_21_snowflake.sql | 32 ++ tests/test_sql_refsols/explode_21_trino.sql | 34 ++ 59 files changed, 1911 insertions(+), 45 deletions(-) create mode 100644 tests/test_plan_refsols/explode_13.txt create mode 100644 tests/test_plan_refsols/explode_14.txt create mode 100644 tests/test_plan_refsols/explode_15.txt create mode 100644 tests/test_plan_refsols/explode_16.txt create mode 100644 tests/test_plan_refsols/explode_17.txt create mode 100644 tests/test_plan_refsols/explode_18.txt create mode 100644 tests/test_plan_refsols/explode_19.txt create mode 100644 tests/test_plan_refsols/explode_20.txt create mode 100644 tests/test_plan_refsols/explode_21.txt create mode 100644 tests/test_sql_refsols/explode_13_databricks.sql create mode 100644 tests/test_sql_refsols/explode_13_duckdb.sql create mode 100644 tests/test_sql_refsols/explode_13_postgres.sql create mode 100644 tests/test_sql_refsols/explode_13_trino.sql create mode 100644 tests/test_sql_refsols/explode_14_databricks.sql create mode 100644 tests/test_sql_refsols/explode_14_duckdb.sql create mode 100644 tests/test_sql_refsols/explode_14_postgres.sql create mode 100644 tests/test_sql_refsols/explode_14_snowflake.sql create mode 100644 tests/test_sql_refsols/explode_14_trino.sql create mode 100644 tests/test_sql_refsols/explode_15_databricks.sql create mode 100644 tests/test_sql_refsols/explode_15_duckdb.sql create mode 100644 tests/test_sql_refsols/explode_15_postgres.sql create mode 100644 tests/test_sql_refsols/explode_15_snowflake.sql create mode 100644 tests/test_sql_refsols/explode_15_trino.sql create mode 100644 tests/test_sql_refsols/explode_16_databricks.sql create mode 100644 tests/test_sql_refsols/explode_16_duckdb.sql create mode 100644 tests/test_sql_refsols/explode_16_postgres.sql create mode 100644 tests/test_sql_refsols/explode_16_snowflake.sql create mode 100644 tests/test_sql_refsols/explode_16_trino.sql create mode 100644 tests/test_sql_refsols/explode_17_databricks.sql create mode 100644 tests/test_sql_refsols/explode_17_duckdb.sql create mode 100644 tests/test_sql_refsols/explode_17_postgres.sql create mode 100644 tests/test_sql_refsols/explode_17_snowflake.sql create mode 100644 tests/test_sql_refsols/explode_17_trino.sql create mode 100644 tests/test_sql_refsols/explode_18_databricks.sql create mode 100644 tests/test_sql_refsols/explode_18_duckdb.sql create mode 100644 tests/test_sql_refsols/explode_18_postgres.sql create mode 100644 tests/test_sql_refsols/explode_18_snowflake.sql create mode 100644 tests/test_sql_refsols/explode_18_trino.sql create mode 100644 tests/test_sql_refsols/explode_19_databricks.sql create mode 100644 tests/test_sql_refsols/explode_19_duckdb.sql create mode 100644 tests/test_sql_refsols/explode_19_postgres.sql create mode 100644 tests/test_sql_refsols/explode_19_snowflake.sql create mode 100644 tests/test_sql_refsols/explode_19_trino.sql create mode 100644 tests/test_sql_refsols/explode_20_databricks.sql create mode 100644 tests/test_sql_refsols/explode_20_duckdb.sql create mode 100644 tests/test_sql_refsols/explode_20_postgres.sql create mode 100644 tests/test_sql_refsols/explode_20_snowflake.sql create mode 100644 tests/test_sql_refsols/explode_20_trino.sql create mode 100644 tests/test_sql_refsols/explode_21_databricks.sql create mode 100644 tests/test_sql_refsols/explode_21_duckdb.sql create mode 100644 tests/test_sql_refsols/explode_21_postgres.sql create mode 100644 tests/test_sql_refsols/explode_21_snowflake.sql create mode 100644 tests/test_sql_refsols/explode_21_trino.sql diff --git a/pydough/sqlglot/transform_bindings/duckdb_transform_bindings.py b/pydough/sqlglot/transform_bindings/duckdb_transform_bindings.py index 5915e42d9..0033b5223 100644 --- a/pydough/sqlglot/transform_bindings/duckdb_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/duckdb_transform_bindings.py @@ -170,10 +170,21 @@ def convert_explode( assert explode_spec.delimiter is not None, ( "Delimiter must be provided for string explode." ) - explode_expr = sqlglot_expressions.Split( - this=explode_expr, - expression=sqlglot_expressions.Literal.string(explode_spec.delimiter), - ) + if explode_spec.delimiter == "": + explode_expr = sqlglot_expressions.Anonymous( + this="REGEXP_SPLIT_TO_ARRAY", + expressions=[ + explode_expr, + sqlglot_expressions.Literal.string(""), + ], + ) + else: + explode_expr = sqlglot_expressions.Split( + this=explode_expr, + expression=sqlglot_expressions.Literal.string( + explode_spec.delimiter + ), + ) explode_args: list[SQLGlotExpression] = [ sqlglot_expressions.Unnest(expressions=[explode_expr]) ] diff --git a/pydough/sqlglot/transform_bindings/postgres_transform_bindings.py b/pydough/sqlglot/transform_bindings/postgres_transform_bindings.py index 1814eaf91..0a94f4f87 100644 --- a/pydough/sqlglot/transform_bindings/postgres_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/postgres_transform_bindings.py @@ -111,13 +111,22 @@ def convert_explode( assert explode_spec.delimiter is not None, ( "Delimiter must be provided for string explode." ) - explode_expr = sqlglot_expressions.Anonymous( - this="STRING_TO_ARRAY", - expressions=[ - explode_expr, - sqlglot_expressions.Literal.string(explode_spec.delimiter), - ], - ) + if explode_spec.delimiter == "": + explode_expr = sqlglot_expressions.Anonymous( + this="REGEXP_SPLIT_TO_ARRAY", + expressions=[ + explode_expr, + sqlglot_expressions.Literal.string(""), + ], + ) + else: + explode_expr = sqlglot_expressions.Anonymous( + this="STRING_TO_ARRAY", + expressions=[ + explode_expr, + sqlglot_expressions.Literal.string(explode_spec.delimiter), + ], + ) explode_op: SQLGlotExpression if val_index is None: explode_op = Unnest( diff --git a/pydough/sqlglot/transform_bindings/sf_transform_bindings.py b/pydough/sqlglot/transform_bindings/sf_transform_bindings.py index 99f94ae85..92d30f1d8 100644 --- a/pydough/sqlglot/transform_bindings/sf_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/sf_transform_bindings.py @@ -288,13 +288,27 @@ def convert_explode( assert ( explode_spec.version == "string" and explode_spec.delimiter is not None ) - explode_op = Anonymous( - this="SPLIT_TO_TABLE", - expressions=[ - explode_expr, - sqlglot_expressions.Literal.string(explode_spec.delimiter), - ], - ) + if explode_spec.delimiter == "": + explode_op = Anonymous( + this="FLATTEN", + expressions=[ + Anonymous( + this="REGEXP_SUBSTR_ALL", + expressions=[ + explode_expr, + sqlglot_expressions.Literal.string(".{1}"), + ], + ) + ], + ) + else: + explode_op = Anonymous( + this="SPLIT_TO_TABLE", + expressions=[ + explode_expr, + sqlglot_expressions.Literal.string(explode_spec.delimiter), + ], + ) result = ( Select() .select(*column_exprs) diff --git a/pydough/unqualified/unqualified_node.py b/pydough/unqualified/unqualified_node.py index 94d6909de..24e0e7b2d 100644 --- a/pydough/unqualified/unqualified_node.py +++ b/pydough/unqualified/unqualified_node.py @@ -487,13 +487,9 @@ def EXPLODE( "Cannot provide a `delimiter` to EXPLODE when `version` is 'array'" ) case "string": - if ( - delimiter is None - or not isinstance(delimiter, str) - or len(delimiter) == 0 - ): + if delimiter is None or not isinstance(delimiter, str): raise PyDoughUnqualifiedException( - "Must provide a non-empty string `delimiter` to EXPLODE when `version` is 'string'" + "Must provide a string `delimiter` to EXPLODE when `version` is 'string'" ) case _: raise PyDoughUnqualifiedException( diff --git a/tests/test_pipeline_tpch_custom.py b/tests/test_pipeline_tpch_custom.py index bd617d2cf..decc9076d 100644 --- a/tests/test_pipeline_tpch_custom.py +++ b/tests/test_pipeline_tpch_custom.py @@ -4109,7 +4109,7 @@ "array_data_01", order_sensitive=True, ignore_array_order=True, - skipped_dialects={"ANSI", "SQLITE"}, + skipped_dialects={"ANSI", "SQLITE", "ORACLE", "MYSQL"}, ), id="array_data_01", ), @@ -4126,7 +4126,7 @@ ), "array_data_02", order_sensitive=True, - skipped_dialects={"ANSI", "SQLITE", "SNOWFLAKE"}, + skipped_dialects={"ANSI", "SQLITE", "SNOWFLAKE", "ORACLE", "MYSQL"}, kwargs={ "array_df": pd.DataFrame( { @@ -4156,7 +4156,7 @@ ), "array_data_03", order_sensitive=True, - skipped_dialects={"ANSI", "SQLITE", "MYSQL", "SNOWFLAKE"}, + skipped_dialects={"ANSI", "SQLITE", "SNOWFLAKE", "ORACLE", "MYSQL"}, kwargs={ "array_df": pd.DataFrame( { @@ -4207,7 +4207,7 @@ ), "array_data_04", order_sensitive=True, - skipped_dialects={"ANSI", "SQLITE", "SNOWFLAKE"}, + skipped_dialects={"ANSI", "SQLITE", "SNOWFLAKE", "ORACLE", "MYSQL"}, kwargs={ "array_df": pd.DataFrame( { @@ -4312,7 +4312,7 @@ ), "explode_01", order_sensitive=True, - skipped_dialects={"ANSI", "SQLITE", "MYSQL"}, + skipped_dialects={"ANSI", "SQLITE", "ORACLE", "MYSQL"}, ), id="explode_01", ), @@ -4345,7 +4345,7 @@ ), "explode_02", order_sensitive=True, - skipped_dialects={"ANSI", "SQLITE", "SNOWFLAKE"}, + skipped_dialects={"ANSI", "SQLITE", "SNOWFLAKE", "ORACLE", "MYSQL"}, kwargs={ "array_df": pd.DataFrame( { @@ -4386,7 +4386,7 @@ ), "explode_03", order_sensitive=True, - skipped_dialects={"ANSI", "SQLITE"}, + skipped_dialects={"ANSI", "SQLITE", "ORACLE", "MYSQL"}, ), id="explode_03", ), @@ -4417,7 +4417,7 @@ ), "explode_04", order_sensitive=True, - skipped_dialects={"ANSI", "SQLITE"}, + skipped_dialects={"ANSI", "SQLITE", "ORACLE", "MYSQL"}, ), id="explode_04", ), @@ -4447,7 +4447,7 @@ ), "explode_05", order_sensitive=True, - skipped_dialects={"ANSI", "SQLITE"}, + skipped_dialects={"ANSI", "SQLITE", "ORACLE", "MYSQL"}, ), id="explode_05", marks=pytest.mark.skip( @@ -4496,7 +4496,7 @@ ), "explode_06", order_sensitive=True, - skipped_dialects={"ANSI", "SQLITE"}, + skipped_dialects={"ANSI", "SQLITE", "ORACLE", "MYSQL"}, ), id="explode_06", marks=pytest.mark.skip( @@ -4536,7 +4536,7 @@ ), "explode_07", order_sensitive=True, - skipped_dialects={"ANSI", "SQLITE"}, + skipped_dialects={"ANSI", "SQLITE", "ORACLE", "MYSQL"}, ), id="explode_07", ), @@ -4664,7 +4664,7 @@ ), "explode_08", order_sensitive=True, - skipped_dialects={"ANSI", "SQLITE"}, + skipped_dialects={"ANSI", "SQLITE", "ORACLE", "MYSQL"}, ), id="explode_08", ), @@ -4735,7 +4735,7 @@ "explode_09", order_sensitive=True, ignore_array_order=True, - skipped_dialects={"ANSI", "SQLITE", "MYSQL"}, + skipped_dialects={"ANSI", "SQLITE", "ORACLE", "MYSQL"}, ), id="explode_09", ), @@ -4790,7 +4790,7 @@ ), "explode_10", order_sensitive=True, - skipped_dialects={"ANSI", "SQLITE"}, + skipped_dialects={"ANSI", "SQLITE", "ORACLE", "MYSQL"}, ), id="explode_10", ), @@ -4823,7 +4823,7 @@ ), "explode_11", order_sensitive=True, - skipped_dialects={"ANSI", "SQLITE"}, + skipped_dialects={"ANSI", "SQLITE", "ORACLE", "MYSQL"}, ), id="explode_11", ), @@ -4845,7 +4845,7 @@ } ), "explode_12", - skipped_dialects={"ANSI", "SQLITE", "SNOWFLAKE"}, + skipped_dialects={"ANSI", "SQLITE", "SNOWFLAKE", "ORACLE", "MYSQL"}, kwargs={ "array_df": pd.DataFrame( { @@ -4857,6 +4857,483 @@ ), id="explode_12", ), + pytest.param( + PyDoughPandasTest( + "array_data = pydough.dataframe_collection(name='tbl', dataframe=array_df, unique_column_names=['key'])\n" + "result = (" + " array_data" + " .CALCULATE(key)" + " .EXPLODE(arr, 'exploded_array', value_name='arr_val', index_name='idx', version='array', filtering=True, is_distinct=False)" + " .PARTITION(name='groups', by=(key, arr_val))" + " .CALCULATE(key, arr_val)" + ")", + "TPCH", + lambda: pd.DataFrame( + { + "key": ["A", "C", "C", "C", "C", "D", "D"], + "arr_val": [1, 2, 3, None, 4, 5, 6], + } + ), + "explode_13", + skipped_dialects={"ANSI", "SQLITE", "SNOWFLAKE", "ORACLE", "MYSQL"}, + kwargs={ + "array_df": pd.DataFrame( + { + "key": ["A", "B", "C", "D"], + "arr": [[1], [], [2, 3, None, 4, None], [5, 6, 5]], + } + ) + }, + ), + id="explode_13", + ), + pytest.param( + PyDoughPandasTest( + "customer_words = (" + " customers" + " .CALCULATE(key)" + " .TOP_K(3, by=key.ASC())" + " .EXPLODE(comment, 'exp1', value_name='val1', index_name='idx1', version='string', delimiter='.')" + " .EXPLODE(val1, 'exp2', value_name='val2', index_name='idx2', version='string', delimiter=',')" + " .EXPLODE(val2, 'exp3', value_name='val3', index_name='idx3', version='string', delimiter=' ')" + " .WHERE(val3 != '')" + ")\n" + "result = (" + " customer_words" + " .BEST(per='customers', by=(idx1.ASC(), idx2.ASC(), idx3.ASC()))" + " .CALCULATE(key, idx1, idx2, idx3, val3)" + ")", + "TPCH", + lambda: pd.DataFrame( + { + "key": [1, 2, 3], + "idx1": [0, 0, 0], + "idx2": [0, 0, 0], + "idx3": [0, 0, 1], + "val3": ["to", "l", "deposits"], + } + ), + "explode_14", + skipped_dialects={"ANSI", "SQLITE", "ORACLE", "MYSQL"}, + ), + id="explode_14", + ), + pytest.param( + PyDoughPandasTest( + "customer_words = (" + " customers" + " .CALCULATE(key)" + " .TOP_K(3, by=key.ASC())" + " .EXPLODE(comment, 'exp1', value_name='val1', index_name='idx1', version='string', delimiter='.')" + " .EXPLODE(val1, 'exp2', value_name='val2', index_name='idx2', version='string', delimiter=',')" + " .EXPLODE(val2, 'exp3', value_name='val3', index_name='idx3', version='string', delimiter=' ')" + " .WHERE(val3 != '')" + ")\n" + "result = (" + " customer_words" + " .BEST(per='customers', by=(idx1.DESC(), idx2.ASC(), idx3.ASC()))" + " .CALCULATE(key, idx1, idx2, idx3, val3)" + ")", + "TPCH", + lambda: pd.DataFrame( + { + "key": [1, 2, 3], + "idx1": [1, 1, 2], + "idx2": [0, 0, 0], + "idx3": [1, 1, 1], + "val3": ["regular", "blithely", "blithely"], + } + ), + "explode_15", + skipped_dialects={"ANSI", "SQLITE", "ORACLE", "MYSQL"}, + ), + id="explode_15", + ), + pytest.param( + PyDoughPandasTest( + "customer_words = (" + " customers" + " .CALCULATE(key)" + " .TOP_K(3, by=key.ASC())" + " .EXPLODE(comment, 'exp1', value_name='val1', index_name='idx1', version='string', delimiter='.')" + " .EXPLODE(val1, 'exp2', value_name='val2', index_name='idx2', version='string', delimiter=',')" + " .EXPLODE(val2, 'exp3', value_name='val3', index_name='idx3', version='string', delimiter=' ')" + " .WHERE(val3 != '')" + ")\n" + "result = (" + " customer_words" + " .BEST(per='customers', by=(idx1.ASC(), idx2.DESC(), idx3.ASC()))" + " .CALCULATE(key, idx1, idx2, idx3, val3)" + ")", + "TPCH", + lambda: pd.DataFrame( + { + "key": [1, 2, 3], + "idx1": [0, 0, 0], + "idx2": [1, 0, 1], + "idx3": [1, 0, 1], + "val3": ["regular", "l", "even"], + } + ), + "explode_16", + skipped_dialects={"ANSI", "SQLITE", "ORACLE", "MYSQL"}, + ), + id="explode_16", + ), + pytest.param( + PyDoughPandasTest( + "customer_words = (" + " customers" + " .CALCULATE(key)" + " .TOP_K(3, by=key.ASC())" + " .EXPLODE(comment, 'exp1', value_name='val1', index_name='idx1', version='string', delimiter='.')" + " .EXPLODE(val1, 'exp2', value_name='val2', index_name='idx2', version='string', delimiter=',')" + " .EXPLODE(val2, 'exp3', value_name='val3', index_name='idx3', version='string', delimiter=' ')" + " .WHERE(val3 != '')" + ")\n" + "result = (" + " customer_words" + " .BEST(per='customers', by=(idx1.DESC(), idx2.DESC(), idx3.ASC()))" + " .CALCULATE(key, idx1, idx2, idx3, val3)" + ")", + "TPCH", + lambda: pd.DataFrame( + { + "key": [1, 2, 3], + "idx1": [1, 1, 2], + "idx2": [1, 0, 0], + "idx3": [1, 1, 1], + "val3": ["ironic", "blithely", "blithely"], + } + ), + "explode_17", + skipped_dialects={"ANSI", "SQLITE", "ORACLE", "MYSQL"}, + ), + id="explode_17", + ), + pytest.param( + PyDoughPandasTest( + "customer_words = (" + " customers" + " .CALCULATE(key)" + " .TOP_K(3, by=key.ASC())" + " .EXPLODE(comment, 'exp1', value_name='val1', index_name='idx1', version='string', delimiter='.')" + " .EXPLODE(val1, 'exp2', value_name='val2', index_name='idx2', version='string', delimiter=',')" + " .EXPLODE(val2, 'exp3', value_name='val3', index_name='idx3', version='string', delimiter=' ')" + " .WHERE(val3 != '')" + ")\n" + "result = (" + " customer_words" + " .BEST(per='exp2', by=idx3.DESC())" + " .CALCULATE(key, idx1, idx2, idx3, val3)" + ")", + "TPCH", + lambda: pd.DataFrame( + { + "key": [1, 1, 1, 1, 2, 2, 3, 3, 3, 3], + "idx1": [0, 0, 1, 1, 0, 1, 0, 0, 1, 2], + "idx2": [0, 1, 0, 1, 0, 0, 0, 1, 0, 0], + "idx3": [2, 2, 1, 4, 1, 6, 4, 2, 4, 4], + "val3": [ + "even", + "platelets", + "regular", + "e", + "accounts", + "caref", + "ironic", + "instructions", + "slyly", + "abov", + ], + } + ), + "explode_18", + skipped_dialects={"ANSI", "SQLITE", "ORACLE", "MYSQL"}, + ), + id="explode_18", + ), + pytest.param( + PyDoughPandasTest( + "customer_words = (" + " customers" + " .CALCULATE(key)" + " .TOP_K(3, by=key.ASC())" + " .EXPLODE(comment, 'exp1', value_name='val1', index_name='idx1', version='string', delimiter='.')" + " .EXPLODE(val1, 'exp2', value_name='val2', index_name='idx2', version='string', delimiter=',')" + " .EXPLODE(val2, 'exp3', value_name='val3', index_name='idx3', version='string', delimiter=' ')" + " .WHERE(val3 != '')" + ")\n" + "result = (" + " customer_words" + " .BEST(per='exp1', by=(idx2.DESC(), idx3.DESC()))" + " .CALCULATE(key, idx1, idx2, idx3, val3)" + ")", + "TPCH", + lambda: pd.DataFrame( + { + "key": [1, 1, 2, 2, 3, 3, 3], + "idx1": [0, 1, 0, 1, 0, 1, 2], + "idx2": [1, 1, 0, 0, 1, 0, 0], + "idx3": [2, 4, 1, 6, 2, 4, 4], + "val3": [ + "platelets", + "e", + "accounts", + "caref", + "instructions", + "slyly", + "abov", + ], + } + ), + "explode_19", + skipped_dialects={"ANSI", "SQLITE", "ORACLE", "MYSQL"}, + ), + id="explode_19", + ), + pytest.param( + PyDoughPandasTest( + "result = (" + " customers" + " .TOP_K(3, by=key.ASC())" + " .EXPLODE(comment, 'exp', value_name='char', index_name='idx', version='string', delimiter='')" + " .WHERE(char != '')" + " .PARTITION(name='chars', by=char)" + " .CALCULATE(char, n=COUNT(exp))" + ")", + "TPCH", + lambda: pd.DataFrame( + { + "char": list(" ,.:abcdefghilnoprstuvxy"), + "exp": [ + 30, + 3, + 4, + 1, + 11, + 4, + 10, + 4, + 27, + 2, + 4, + 5, + 14, + 16, + 12, + 13, + 5, + 11, + 14, + 19, + 5, + 4, + 2, + 7, + ], + } + ), + "explode_20", + skipped_dialects={"ANSI", "SQLITE", "ORACLE", "MYSQL"}, + ), + id="explode_20", + ), + pytest.param( + PyDoughPandasTest( + "result = (" + " customers" + " .CALCULATE(key)" + " .TOP_K(3, by=key.ASC())" + " .EXPLODE(comment, 'exp1', value_name='val1', index_name='idx1', version='string', delimiter='.')" + " .EXPLODE(val1, 'exp2', value_name='val2', index_name='idx2', version='string', delimiter=',')" + " .EXPLODE(val2, 'exp3', value_name='val3', index_name='idx3', version='string', delimiter=' ')" + " .EXPLODE(val3, 'expr4', value_name='val4', index_name='idx4', version='string', delimiter='')" + " .WHERE(val4 != '')" + " .BEST(per='exp3', by=idx4.ASC())" + " .CALCULATE(key, idx1, idx2, idx3, idx4, val4)" + ")", + "TPCH", + lambda: pd.DataFrame( + { + "key": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + ], + "idx1": [ + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 2, + 2, + 2, + 2, + ], + "idx2": [ + 0, + 0, + 0, + 1, + 1, + 0, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + ], + "idx3": [ + 0, + 1, + 2, + 1, + 2, + 1, + 1, + 2, + 3, + 4, + 0, + 1, + 1, + 2, + 3, + 4, + 5, + 6, + 1, + 2, + 3, + 4, + 1, + 2, + 1, + 2, + 3, + 4, + 1, + 2, + 3, + 4, + ], + "idx4": [0] * 32, + "val4": [ + "t", + "t", + "e", + "r", + "p", + "r", + "i", + "e", + "n", + "e", + "l", + "a", + "b", + "i", + "t", + "i", + "b", + "c", + "d", + "e", + "s", + "i", + "e", + "i", + "e", + "f", + "d", + "s", + "b", + "e", + "a", + "a", + ], + } + ), + "explode_21", + skipped_dialects={"ANSI", "SQLITE", "ORACLE", "MYSQL"}, + ), + id="explode_21", + ), pytest.param( PyDoughPandasTest( simple_range_1, diff --git a/tests/test_plan_refsols/explode_13.txt b/tests/test_plan_refsols/explode_13.txt new file mode 100644 index 000000000..c5e377b66 --- /dev/null +++ b/tests/test_plan_refsols/explode_13.txt @@ -0,0 +1,4 @@ +ROOT(columns=[('key', key), ('arr_val', arr_val)], orderings=[]) + AGGREGATE(keys={'arr_val': arr_val, 'key': key}, aggregations={}) + EXPLODE(arr, value_name='arr_val', index_name='idx', version='array', filtering=True, is_distinct=False, columns={'arr_val': arr_val, 'idx': idx, 'key': key}) + GENERATED_TABLE(DataframeCollection(name='tbl', shape=(4, 2), columns=['key', 'arr'])) diff --git a/tests/test_plan_refsols/explode_14.txt b/tests/test_plan_refsols/explode_14.txt new file mode 100644 index 000000000..ff3896867 --- /dev/null +++ b/tests/test_plan_refsols/explode_14.txt @@ -0,0 +1,8 @@ +ROOT(columns=[('key', key), ('idx1', idx1), ('idx2', idx2), ('idx3', idx3), ('val3', val3)], orderings=[]) + FILTER(condition=1:numeric == RANKING(args=[], partition=[key], order=[(idx1):asc_last, (idx2):asc_last, (idx3):asc_last], allow_ties=False), columns={'idx1': idx1, 'idx2': idx2, 'idx3': idx3, 'key': key, 'val3': val3}) + FILTER(condition=val3 != '':string, columns={'idx1': idx1, 'idx2': idx2, 'idx3': idx3, 'key': key, 'val3': val3}) + EXPLODE(val2, value_name='val3', index_name='idx3', version='string', delimiter=' ', filtering=True, is_distinct=False, columns={'idx1': idx1, 'idx2': idx2, 'idx3': idx3, 'key': key, 'val3': val3}) + EXPLODE(val1, value_name='val2', index_name='idx2', version='string', delimiter=',', filtering=True, is_distinct=False, columns={'idx1': idx1, 'idx2': idx2, 'key': key, 'val2': val2}) + EXPLODE(comment, value_name='val1', index_name='idx1', version='string', delimiter='.', filtering=True, is_distinct=False, columns={'idx1': idx1, 'key': key, 'val1': val1}) + LIMIT(limit=3:numeric, columns={'comment': comment, 'key': key}, orderings=[(key):asc_first]) + SCAN(table=tpch.CUSTOMER, columns={'comment': c_comment, 'key': c_custkey}) diff --git a/tests/test_plan_refsols/explode_15.txt b/tests/test_plan_refsols/explode_15.txt new file mode 100644 index 000000000..0c8dfc866 --- /dev/null +++ b/tests/test_plan_refsols/explode_15.txt @@ -0,0 +1,8 @@ +ROOT(columns=[('key', key), ('idx1', idx1), ('idx2', idx2), ('idx3', idx3), ('val3', val3)], orderings=[]) + FILTER(condition=1:numeric == RANKING(args=[], partition=[key], order=[(idx1):desc_first, (idx2):asc_last, (idx3):asc_last], allow_ties=False), columns={'idx1': idx1, 'idx2': idx2, 'idx3': idx3, 'key': key, 'val3': val3}) + FILTER(condition=val3 != '':string, columns={'idx1': idx1, 'idx2': idx2, 'idx3': idx3, 'key': key, 'val3': val3}) + EXPLODE(val2, value_name='val3', index_name='idx3', version='string', delimiter=' ', filtering=True, is_distinct=False, columns={'idx1': idx1, 'idx2': idx2, 'idx3': idx3, 'key': key, 'val3': val3}) + EXPLODE(val1, value_name='val2', index_name='idx2', version='string', delimiter=',', filtering=True, is_distinct=False, columns={'idx1': idx1, 'idx2': idx2, 'key': key, 'val2': val2}) + EXPLODE(comment, value_name='val1', index_name='idx1', version='string', delimiter='.', filtering=True, is_distinct=False, columns={'idx1': idx1, 'key': key, 'val1': val1}) + LIMIT(limit=3:numeric, columns={'comment': comment, 'key': key}, orderings=[(key):asc_first]) + SCAN(table=tpch.CUSTOMER, columns={'comment': c_comment, 'key': c_custkey}) diff --git a/tests/test_plan_refsols/explode_16.txt b/tests/test_plan_refsols/explode_16.txt new file mode 100644 index 000000000..c6759c96a --- /dev/null +++ b/tests/test_plan_refsols/explode_16.txt @@ -0,0 +1,8 @@ +ROOT(columns=[('key', key), ('idx1', idx1), ('idx2', idx2), ('idx3', idx3), ('val3', val3)], orderings=[]) + FILTER(condition=1:numeric == RANKING(args=[], partition=[key], order=[(idx1):asc_last, (idx2):desc_first, (idx3):asc_last], allow_ties=False), columns={'idx1': idx1, 'idx2': idx2, 'idx3': idx3, 'key': key, 'val3': val3}) + FILTER(condition=val3 != '':string, columns={'idx1': idx1, 'idx2': idx2, 'idx3': idx3, 'key': key, 'val3': val3}) + EXPLODE(val2, value_name='val3', index_name='idx3', version='string', delimiter=' ', filtering=True, is_distinct=False, columns={'idx1': idx1, 'idx2': idx2, 'idx3': idx3, 'key': key, 'val3': val3}) + EXPLODE(val1, value_name='val2', index_name='idx2', version='string', delimiter=',', filtering=True, is_distinct=False, columns={'idx1': idx1, 'idx2': idx2, 'key': key, 'val2': val2}) + EXPLODE(comment, value_name='val1', index_name='idx1', version='string', delimiter='.', filtering=True, is_distinct=False, columns={'idx1': idx1, 'key': key, 'val1': val1}) + LIMIT(limit=3:numeric, columns={'comment': comment, 'key': key}, orderings=[(key):asc_first]) + SCAN(table=tpch.CUSTOMER, columns={'comment': c_comment, 'key': c_custkey}) diff --git a/tests/test_plan_refsols/explode_17.txt b/tests/test_plan_refsols/explode_17.txt new file mode 100644 index 000000000..66c6f4486 --- /dev/null +++ b/tests/test_plan_refsols/explode_17.txt @@ -0,0 +1,8 @@ +ROOT(columns=[('key', key), ('idx1', idx1), ('idx2', idx2), ('idx3', idx3), ('val3', val3)], orderings=[]) + FILTER(condition=1:numeric == RANKING(args=[], partition=[key], order=[(idx1):desc_first, (idx2):desc_first, (idx3):asc_last], allow_ties=False), columns={'idx1': idx1, 'idx2': idx2, 'idx3': idx3, 'key': key, 'val3': val3}) + FILTER(condition=val3 != '':string, columns={'idx1': idx1, 'idx2': idx2, 'idx3': idx3, 'key': key, 'val3': val3}) + EXPLODE(val2, value_name='val3', index_name='idx3', version='string', delimiter=' ', filtering=True, is_distinct=False, columns={'idx1': idx1, 'idx2': idx2, 'idx3': idx3, 'key': key, 'val3': val3}) + EXPLODE(val1, value_name='val2', index_name='idx2', version='string', delimiter=',', filtering=True, is_distinct=False, columns={'idx1': idx1, 'idx2': idx2, 'key': key, 'val2': val2}) + EXPLODE(comment, value_name='val1', index_name='idx1', version='string', delimiter='.', filtering=True, is_distinct=False, columns={'idx1': idx1, 'key': key, 'val1': val1}) + LIMIT(limit=3:numeric, columns={'comment': comment, 'key': key}, orderings=[(key):asc_first]) + SCAN(table=tpch.CUSTOMER, columns={'comment': c_comment, 'key': c_custkey}) diff --git a/tests/test_plan_refsols/explode_18.txt b/tests/test_plan_refsols/explode_18.txt new file mode 100644 index 000000000..287ca80ca --- /dev/null +++ b/tests/test_plan_refsols/explode_18.txt @@ -0,0 +1,8 @@ +ROOT(columns=[('key', key), ('idx1', idx1), ('idx2', idx2), ('idx3', idx3), ('val3', val3)], orderings=[]) + FILTER(condition=1:numeric == RANKING(args=[], partition=[idx1, key, idx2], order=[(idx3):desc_first], allow_ties=False), columns={'idx1': idx1, 'idx2': idx2, 'idx3': idx3, 'key': key, 'val3': val3}) + FILTER(condition=val3 != '':string, columns={'idx1': idx1, 'idx2': idx2, 'idx3': idx3, 'key': key, 'val3': val3}) + EXPLODE(val2, value_name='val3', index_name='idx3', version='string', delimiter=' ', filtering=True, is_distinct=False, columns={'idx1': idx1, 'idx2': idx2, 'idx3': idx3, 'key': key, 'val3': val3}) + EXPLODE(val1, value_name='val2', index_name='idx2', version='string', delimiter=',', filtering=True, is_distinct=False, columns={'idx1': idx1, 'idx2': idx2, 'key': key, 'val2': val2}) + EXPLODE(comment, value_name='val1', index_name='idx1', version='string', delimiter='.', filtering=True, is_distinct=False, columns={'idx1': idx1, 'key': key, 'val1': val1}) + LIMIT(limit=3:numeric, columns={'comment': comment, 'key': key}, orderings=[(key):asc_first]) + SCAN(table=tpch.CUSTOMER, columns={'comment': c_comment, 'key': c_custkey}) diff --git a/tests/test_plan_refsols/explode_19.txt b/tests/test_plan_refsols/explode_19.txt new file mode 100644 index 000000000..df3525f4f --- /dev/null +++ b/tests/test_plan_refsols/explode_19.txt @@ -0,0 +1,8 @@ +ROOT(columns=[('key', key), ('idx1', idx1), ('idx2', idx2), ('idx3', idx3), ('val3', val3)], orderings=[]) + FILTER(condition=1:numeric == RANKING(args=[], partition=[key, idx1], order=[(idx2):desc_first, (idx3):desc_first], allow_ties=False), columns={'idx1': idx1, 'idx2': idx2, 'idx3': idx3, 'key': key, 'val3': val3}) + FILTER(condition=val3 != '':string, columns={'idx1': idx1, 'idx2': idx2, 'idx3': idx3, 'key': key, 'val3': val3}) + EXPLODE(val2, value_name='val3', index_name='idx3', version='string', delimiter=' ', filtering=True, is_distinct=False, columns={'idx1': idx1, 'idx2': idx2, 'idx3': idx3, 'key': key, 'val3': val3}) + EXPLODE(val1, value_name='val2', index_name='idx2', version='string', delimiter=',', filtering=True, is_distinct=False, columns={'idx1': idx1, 'idx2': idx2, 'key': key, 'val2': val2}) + EXPLODE(comment, value_name='val1', index_name='idx1', version='string', delimiter='.', filtering=True, is_distinct=False, columns={'idx1': idx1, 'key': key, 'val1': val1}) + LIMIT(limit=3:numeric, columns={'comment': comment, 'key': key}, orderings=[(key):asc_first]) + SCAN(table=tpch.CUSTOMER, columns={'comment': c_comment, 'key': c_custkey}) diff --git a/tests/test_plan_refsols/explode_20.txt b/tests/test_plan_refsols/explode_20.txt new file mode 100644 index 000000000..e3256c8a9 --- /dev/null +++ b/tests/test_plan_refsols/explode_20.txt @@ -0,0 +1,6 @@ +ROOT(columns=[('char', char), ('n', n_rows)], orderings=[]) + AGGREGATE(keys={'char': char}, aggregations={'n_rows': COUNT()}) + FILTER(condition=char != '':string, columns={'char': char}) + EXPLODE(comment, value_name='char', index_name='idx', version='string', delimiter='', filtering=True, is_distinct=False, columns={'char': char, 'idx': idx}) + LIMIT(limit=3:numeric, columns={'comment': comment}, orderings=[(key):asc_first]) + SCAN(table=tpch.CUSTOMER, columns={'comment': c_comment, 'key': c_custkey}) diff --git a/tests/test_plan_refsols/explode_21.txt b/tests/test_plan_refsols/explode_21.txt new file mode 100644 index 000000000..771d51764 --- /dev/null +++ b/tests/test_plan_refsols/explode_21.txt @@ -0,0 +1,9 @@ +ROOT(columns=[('key', key), ('idx1', idx1), ('idx2', idx2), ('idx3', idx3), ('idx4', idx4), ('val4', val4)], orderings=[]) + FILTER(condition=1:numeric == RANKING(args=[], partition=[idx2, idx1, key, idx3], order=[(idx4):asc_last], allow_ties=False), columns={'idx1': idx1, 'idx2': idx2, 'idx3': idx3, 'idx4': idx4, 'key': key, 'val4': val4}) + FILTER(condition=val4 != '':string, columns={'idx1': idx1, 'idx2': idx2, 'idx3': idx3, 'idx4': idx4, 'key': key, 'val4': val4}) + EXPLODE(val3, value_name='val4', index_name='idx4', version='string', delimiter='', filtering=True, is_distinct=False, columns={'idx1': idx1, 'idx2': idx2, 'idx3': idx3, 'idx4': idx4, 'key': key, 'val4': val4}) + EXPLODE(val2, value_name='val3', index_name='idx3', version='string', delimiter=' ', filtering=True, is_distinct=False, columns={'idx1': idx1, 'idx2': idx2, 'idx3': idx3, 'key': key, 'val3': val3}) + EXPLODE(val1, value_name='val2', index_name='idx2', version='string', delimiter=',', filtering=True, is_distinct=False, columns={'idx1': idx1, 'idx2': idx2, 'key': key, 'val2': val2}) + EXPLODE(comment, value_name='val1', index_name='idx1', version='string', delimiter='.', filtering=True, is_distinct=False, columns={'idx1': idx1, 'key': key, 'val1': val1}) + LIMIT(limit=3:numeric, columns={'comment': comment, 'key': key}, orderings=[(key):asc_first]) + SCAN(table=tpch.CUSTOMER, columns={'comment': c_comment, 'key': c_custkey}) diff --git a/tests/test_qualification_errors.py b/tests/test_qualification_errors.py index e99520aec..3641d2df3 100644 --- a/tests/test_qualification_errors.py +++ b/tests/test_qualification_errors.py @@ -227,17 +227,17 @@ ), pytest.param( "result = nations.EXPLODE(name, 'names', value_name='v', index_name='i', version='string')", - "Must provide a non-empty string `delimiter` to EXPLODE when `version` is 'string'", + "Must provide a string `delimiter` to EXPLODE when `version` is 'string'", id="bad_explode_06", ), pytest.param( - "result = nations.EXPLODE(name, 'names', value_name='v', index_name='i', version='string', delimiter='')", - "Must provide a non-empty string `delimiter` to EXPLODE when `version` is 'string'", + "result = nations.EXPLODE(name, 'names', value_name='v', index_name='i', version='string', delimiter=['x'])", + "Must provide a string `delimiter` to EXPLODE when `version` is 'string'", id="bad_explode_07", ), pytest.param( "result = nations.EXPLODE(name, 'names', value_name='v', index_name='i', version='string', delimiter=42)", - "Must provide a non-empty string `delimiter` to EXPLODE when `version` is 'string'", + "Must provide a string `delimiter` to EXPLODE when `version` is 'string'", id="bad_explode_08", ), pytest.param( @@ -257,7 +257,7 @@ ), pytest.param( "result = nations.EXPLODE(name, 'names', value_name='v', index_name='i', version='string', delimiter=0)", - "Must provide a non-empty string `delimiter` to EXPLODE when `version` is 'string'", + "Must provide a string `delimiter` to EXPLODE when `version` is 'string'", id="bad_explode_12", ), pytest.param( diff --git a/tests/test_sql_refsols/explode_13_databricks.sql b/tests/test_sql_refsols/explode_13_databricks.sql new file mode 100644 index 000000000..5f30623a9 --- /dev/null +++ b/tests/test_sql_refsols/explode_13_databricks.sql @@ -0,0 +1,9 @@ +SELECT DISTINCT + tbl.key, + _s0.val AS arr_val +FROM VALUES + ('A', ARRAY(1)), + ('B', ARRAY()), + ('C', ARRAY(2, 3, NULL, 4, NULL)), + ('D', ARRAY(5, 6, 5)) AS tbl(key, arr) +CROSS JOIN LATERAL POSEXPLODE(tbl.arr) AS _s0(idx, val) diff --git a/tests/test_sql_refsols/explode_13_duckdb.sql b/tests/test_sql_refsols/explode_13_duckdb.sql new file mode 100644 index 000000000..e8b6264cb --- /dev/null +++ b/tests/test_sql_refsols/explode_13_duckdb.sql @@ -0,0 +1,13 @@ +SELECT DISTINCT + tbl.key, + _s0.val AS arr_val +FROM (VALUES + ('A', [1]), + ('B', []), + ('C', [2, 3, NULL, 4, NULL]), + ('D', [5, 6, 5])) AS tbl(key, arr) +CROSS JOIN LATERAL ( + SELECT + UNNEST(tbl.arr) AS _col_0, + GENERATE_SUBSCRIPTS(tbl.arr, 1) - 1 AS _col_1 +) AS _s0(val, idx) diff --git a/tests/test_sql_refsols/explode_13_postgres.sql b/tests/test_sql_refsols/explode_13_postgres.sql new file mode 100644 index 000000000..c4b62f394 --- /dev/null +++ b/tests/test_sql_refsols/explode_13_postgres.sql @@ -0,0 +1,11 @@ +SELECT DISTINCT + tbl.key, + _s0.val AS arr_val +FROM (VALUES + ('A', ARRAY[1]), + ('B', ( + ARRAY[0] + )[1 : 0]), + ('C', ARRAY[2, 3, NULL, 4, NULL]), + ('D', ARRAY[5, 6, 5])) AS tbl(key, arr) +CROSS JOIN LATERAL UNNEST(tbl.arr) WITH ORDINALITY AS _s0(val, idx) diff --git a/tests/test_sql_refsols/explode_13_trino.sql b/tests/test_sql_refsols/explode_13_trino.sql new file mode 100644 index 000000000..b37549864 --- /dev/null +++ b/tests/test_sql_refsols/explode_13_trino.sql @@ -0,0 +1,9 @@ +SELECT DISTINCT + tbl.key, + _s0.val AS arr_val +FROM (VALUES + ('A', ARRAY[1]), + ('B', ARRAY[]), + ('C', ARRAY[2, 3, NULL, 4, NULL]), + ('D', ARRAY[5, 6, 5])) AS tbl(key, arr) +CROSS JOIN UNNEST(tbl.arr) WITH ORDINALITY AS _s0(val, idx) diff --git a/tests/test_sql_refsols/explode_14_databricks.sql b/tests/test_sql_refsols/explode_14_databricks.sql new file mode 100644 index 000000000..8fc5e40b0 --- /dev/null +++ b/tests/test_sql_refsols/explode_14_databricks.sql @@ -0,0 +1,30 @@ +WITH _q_0 AS ( + SELECT + c_comment AS comment, + c_custkey AS key + FROM tpch.customer + ORDER BY + 2 + LIMIT 3 +), _t0 AS ( + SELECT + _s0.idx AS idx1, + _s1.idx AS idx2, + _s2.idx AS idx3, + _q_0.key, + _s2.val AS val3 + FROM _q_0 AS _q_0 + CROSS JOIN LATERAL POSEXPLODE(SPLIT(_q_0.comment, '\\Q.\\E')) AS _s0(idx, val) + CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s0.val, '\\Q,\\E')) AS _s1(idx, val) + JOIN LATERAL POSEXPLODE(SPLIT(_s1.val, '\\Q \\E')) AS _s2(idx, val) + ON _s2.val <> '' + QUALIFY + ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.idx NULLS LAST, _s1.idx NULLS LAST, _s2.idx NULLS LAST) = 1 +) +SELECT + key, + idx1, + idx2, + idx3, + val3 +FROM _t0 diff --git a/tests/test_sql_refsols/explode_14_duckdb.sql b/tests/test_sql_refsols/explode_14_duckdb.sql new file mode 100644 index 000000000..3d8b4abd9 --- /dev/null +++ b/tests/test_sql_refsols/explode_14_duckdb.sql @@ -0,0 +1,42 @@ +WITH _q_0 AS ( + SELECT + c_comment AS comment, + c_custkey AS key + FROM tpch.customer + ORDER BY + 2 NULLS FIRST + LIMIT 3 +), _t0 AS ( + SELECT + _s0.idx AS idx1, + _s1.idx AS idx2, + _s2.idx AS idx3, + _q_0.key, + _s2.val AS val3 + FROM _q_0 AS _q_0 + CROSS JOIN LATERAL ( + SELECT + UNNEST(STR_SPLIT(_q_0.comment, '.')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_q_0.comment, '.'), 1) - 1 AS _col_1 + ) AS _s0(val, idx) + CROSS JOIN LATERAL ( + SELECT + UNNEST(STR_SPLIT(_s0.val, ',')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_s0.val, ','), 1) - 1 AS _col_1 + ) AS _s1(val, idx) + JOIN LATERAL ( + SELECT + UNNEST(STR_SPLIT(_s1.val, ' ')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_s1.val, ' '), 1) - 1 AS _col_1 + ) AS _s2(val, idx) + ON _s2.val <> '' + QUALIFY + ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.idx, _s1.idx, _s2.idx) = 1 +) +SELECT + key, + idx1, + idx2, + idx3, + val3 +FROM _t0 diff --git a/tests/test_sql_refsols/explode_14_postgres.sql b/tests/test_sql_refsols/explode_14_postgres.sql new file mode 100644 index 000000000..2c2513ff4 --- /dev/null +++ b/tests/test_sql_refsols/explode_14_postgres.sql @@ -0,0 +1,31 @@ +WITH _q_0 AS ( + SELECT + c_comment AS comment, + c_custkey AS key + FROM tpch.customer + ORDER BY + 2 NULLS FIRST + LIMIT 3 +), _t AS ( + SELECT + _s0.idx - 1 AS idx1, + _s1.idx - 1 AS idx2, + _s2.idx - 1 AS idx3, + _q_0.key, + _s2.val AS val3, + ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.idx - 1, _s1.idx - 1, _s2.idx - 1) AS _w + FROM _q_0 AS _q_0 + CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) + CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx) + JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) + ON _s2.val <> '' +) +SELECT + key, + idx1, + idx2, + idx3, + val3 +FROM _t +WHERE + _w = 1 diff --git a/tests/test_sql_refsols/explode_14_snowflake.sql b/tests/test_sql_refsols/explode_14_snowflake.sql new file mode 100644 index 000000000..8544d8bb5 --- /dev/null +++ b/tests/test_sql_refsols/explode_14_snowflake.sql @@ -0,0 +1,30 @@ +WITH _q_0 AS ( + SELECT + c_comment AS comment, + c_custkey AS key + FROM tpch.customer + ORDER BY + 2 NULLS FIRST + LIMIT 3 +), _t0 AS ( + SELECT + _s0.index - 1 AS idx1, + _s1.index - 1 AS idx2, + _s2.index - 1 AS idx3, + _q_0.key, + _s2.value AS val3 + FROM _q_0 AS _q_0 + CROSS JOIN LATERAL SPLIT_TO_TABLE(_q_0.comment, '.') AS _s0 + CROSS JOIN LATERAL SPLIT_TO_TABLE(_s0.value, ',') AS _s1 + JOIN LATERAL SPLIT_TO_TABLE(_s1.value, ' ') AS _s2 + ON _s2.value <> '' + QUALIFY + ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.index - 1, _s1.index - 1, _s2.index - 1) = 1 +) +SELECT + key, + idx1, + idx2, + idx3, + val3 +FROM _t0 diff --git a/tests/test_sql_refsols/explode_14_trino.sql b/tests/test_sql_refsols/explode_14_trino.sql new file mode 100644 index 000000000..834168167 --- /dev/null +++ b/tests/test_sql_refsols/explode_14_trino.sql @@ -0,0 +1,31 @@ +WITH _q_0 AS ( + SELECT + c_comment AS comment, + c_custkey AS key + FROM tpch.customer + ORDER BY + 2 NULLS FIRST + LIMIT 3 +), _t AS ( + SELECT + _s0.idx - 1 AS idx1, + _s1.idx - 1 AS idx2, + _s2.idx - 1 AS idx3, + _q_0.key, + _s2.val AS val3, + ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.idx - 1, _s1.idx - 1, _s2.idx - 1) AS _w + FROM _q_0 AS _q_0 + CROSS JOIN UNNEST(SPLIT(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) + CROSS JOIN UNNEST(SPLIT(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx) + JOIN UNNEST(SPLIT(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) + ON _s2.val <> '' +) +SELECT + key, + idx1, + idx2, + idx3, + val3 +FROM _t +WHERE + _w = 1 diff --git a/tests/test_sql_refsols/explode_15_databricks.sql b/tests/test_sql_refsols/explode_15_databricks.sql new file mode 100644 index 000000000..a7e97c983 --- /dev/null +++ b/tests/test_sql_refsols/explode_15_databricks.sql @@ -0,0 +1,30 @@ +WITH _q_0 AS ( + SELECT + c_comment AS comment, + c_custkey AS key + FROM tpch.customer + ORDER BY + 2 + LIMIT 3 +), _t0 AS ( + SELECT + _s0.idx AS idx1, + _s1.idx AS idx2, + _s2.idx AS idx3, + _q_0.key, + _s2.val AS val3 + FROM _q_0 AS _q_0 + CROSS JOIN LATERAL POSEXPLODE(SPLIT(_q_0.comment, '\\Q.\\E')) AS _s0(idx, val) + CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s0.val, '\\Q,\\E')) AS _s1(idx, val) + JOIN LATERAL POSEXPLODE(SPLIT(_s1.val, '\\Q \\E')) AS _s2(idx, val) + ON _s2.val <> '' + QUALIFY + ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.idx DESC NULLS FIRST, _s1.idx NULLS LAST, _s2.idx NULLS LAST) = 1 +) +SELECT + key, + idx1, + idx2, + idx3, + val3 +FROM _t0 diff --git a/tests/test_sql_refsols/explode_15_duckdb.sql b/tests/test_sql_refsols/explode_15_duckdb.sql new file mode 100644 index 000000000..6742661c1 --- /dev/null +++ b/tests/test_sql_refsols/explode_15_duckdb.sql @@ -0,0 +1,42 @@ +WITH _q_0 AS ( + SELECT + c_comment AS comment, + c_custkey AS key + FROM tpch.customer + ORDER BY + 2 NULLS FIRST + LIMIT 3 +), _t0 AS ( + SELECT + _s0.idx AS idx1, + _s1.idx AS idx2, + _s2.idx AS idx3, + _q_0.key, + _s2.val AS val3 + FROM _q_0 AS _q_0 + CROSS JOIN LATERAL ( + SELECT + UNNEST(STR_SPLIT(_q_0.comment, '.')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_q_0.comment, '.'), 1) - 1 AS _col_1 + ) AS _s0(val, idx) + CROSS JOIN LATERAL ( + SELECT + UNNEST(STR_SPLIT(_s0.val, ',')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_s0.val, ','), 1) - 1 AS _col_1 + ) AS _s1(val, idx) + JOIN LATERAL ( + SELECT + UNNEST(STR_SPLIT(_s1.val, ' ')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_s1.val, ' '), 1) - 1 AS _col_1 + ) AS _s2(val, idx) + ON _s2.val <> '' + QUALIFY + ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.idx DESC NULLS FIRST, _s1.idx, _s2.idx) = 1 +) +SELECT + key, + idx1, + idx2, + idx3, + val3 +FROM _t0 diff --git a/tests/test_sql_refsols/explode_15_postgres.sql b/tests/test_sql_refsols/explode_15_postgres.sql new file mode 100644 index 000000000..31cf22fd5 --- /dev/null +++ b/tests/test_sql_refsols/explode_15_postgres.sql @@ -0,0 +1,31 @@ +WITH _q_0 AS ( + SELECT + c_comment AS comment, + c_custkey AS key + FROM tpch.customer + ORDER BY + 2 NULLS FIRST + LIMIT 3 +), _t AS ( + SELECT + _s0.idx - 1 AS idx1, + _s1.idx - 1 AS idx2, + _s2.idx - 1 AS idx3, + _q_0.key, + _s2.val AS val3, + ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.idx - 1 DESC, _s1.idx - 1, _s2.idx - 1) AS _w + FROM _q_0 AS _q_0 + CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) + CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx) + JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) + ON _s2.val <> '' +) +SELECT + key, + idx1, + idx2, + idx3, + val3 +FROM _t +WHERE + _w = 1 diff --git a/tests/test_sql_refsols/explode_15_snowflake.sql b/tests/test_sql_refsols/explode_15_snowflake.sql new file mode 100644 index 000000000..f4dd6145f --- /dev/null +++ b/tests/test_sql_refsols/explode_15_snowflake.sql @@ -0,0 +1,30 @@ +WITH _q_0 AS ( + SELECT + c_comment AS comment, + c_custkey AS key + FROM tpch.customer + ORDER BY + 2 NULLS FIRST + LIMIT 3 +), _t0 AS ( + SELECT + _s0.index - 1 AS idx1, + _s1.index - 1 AS idx2, + _s2.index - 1 AS idx3, + _q_0.key, + _s2.value AS val3 + FROM _q_0 AS _q_0 + CROSS JOIN LATERAL SPLIT_TO_TABLE(_q_0.comment, '.') AS _s0 + CROSS JOIN LATERAL SPLIT_TO_TABLE(_s0.value, ',') AS _s1 + JOIN LATERAL SPLIT_TO_TABLE(_s1.value, ' ') AS _s2 + ON _s2.value <> '' + QUALIFY + ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.index - 1 DESC, _s1.index - 1, _s2.index - 1) = 1 +) +SELECT + key, + idx1, + idx2, + idx3, + val3 +FROM _t0 diff --git a/tests/test_sql_refsols/explode_15_trino.sql b/tests/test_sql_refsols/explode_15_trino.sql new file mode 100644 index 000000000..f59f5839a --- /dev/null +++ b/tests/test_sql_refsols/explode_15_trino.sql @@ -0,0 +1,31 @@ +WITH _q_0 AS ( + SELECT + c_comment AS comment, + c_custkey AS key + FROM tpch.customer + ORDER BY + 2 NULLS FIRST + LIMIT 3 +), _t AS ( + SELECT + _s0.idx - 1 AS idx1, + _s1.idx - 1 AS idx2, + _s2.idx - 1 AS idx3, + _q_0.key, + _s2.val AS val3, + ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.idx - 1 DESC NULLS FIRST, _s1.idx - 1, _s2.idx - 1) AS _w + FROM _q_0 AS _q_0 + CROSS JOIN UNNEST(SPLIT(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) + CROSS JOIN UNNEST(SPLIT(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx) + JOIN UNNEST(SPLIT(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) + ON _s2.val <> '' +) +SELECT + key, + idx1, + idx2, + idx3, + val3 +FROM _t +WHERE + _w = 1 diff --git a/tests/test_sql_refsols/explode_16_databricks.sql b/tests/test_sql_refsols/explode_16_databricks.sql new file mode 100644 index 000000000..8f2fa19c1 --- /dev/null +++ b/tests/test_sql_refsols/explode_16_databricks.sql @@ -0,0 +1,30 @@ +WITH _q_0 AS ( + SELECT + c_comment AS comment, + c_custkey AS key + FROM tpch.customer + ORDER BY + 2 + LIMIT 3 +), _t0 AS ( + SELECT + _s0.idx AS idx1, + _s1.idx AS idx2, + _s2.idx AS idx3, + _q_0.key, + _s2.val AS val3 + FROM _q_0 AS _q_0 + CROSS JOIN LATERAL POSEXPLODE(SPLIT(_q_0.comment, '\\Q.\\E')) AS _s0(idx, val) + CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s0.val, '\\Q,\\E')) AS _s1(idx, val) + JOIN LATERAL POSEXPLODE(SPLIT(_s1.val, '\\Q \\E')) AS _s2(idx, val) + ON _s2.val <> '' + QUALIFY + ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.idx NULLS LAST, _s1.idx DESC NULLS FIRST, _s2.idx NULLS LAST) = 1 +) +SELECT + key, + idx1, + idx2, + idx3, + val3 +FROM _t0 diff --git a/tests/test_sql_refsols/explode_16_duckdb.sql b/tests/test_sql_refsols/explode_16_duckdb.sql new file mode 100644 index 000000000..11e2bdcfc --- /dev/null +++ b/tests/test_sql_refsols/explode_16_duckdb.sql @@ -0,0 +1,42 @@ +WITH _q_0 AS ( + SELECT + c_comment AS comment, + c_custkey AS key + FROM tpch.customer + ORDER BY + 2 NULLS FIRST + LIMIT 3 +), _t0 AS ( + SELECT + _s0.idx AS idx1, + _s1.idx AS idx2, + _s2.idx AS idx3, + _q_0.key, + _s2.val AS val3 + FROM _q_0 AS _q_0 + CROSS JOIN LATERAL ( + SELECT + UNNEST(STR_SPLIT(_q_0.comment, '.')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_q_0.comment, '.'), 1) - 1 AS _col_1 + ) AS _s0(val, idx) + CROSS JOIN LATERAL ( + SELECT + UNNEST(STR_SPLIT(_s0.val, ',')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_s0.val, ','), 1) - 1 AS _col_1 + ) AS _s1(val, idx) + JOIN LATERAL ( + SELECT + UNNEST(STR_SPLIT(_s1.val, ' ')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_s1.val, ' '), 1) - 1 AS _col_1 + ) AS _s2(val, idx) + ON _s2.val <> '' + QUALIFY + ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.idx, _s1.idx DESC NULLS FIRST, _s2.idx) = 1 +) +SELECT + key, + idx1, + idx2, + idx3, + val3 +FROM _t0 diff --git a/tests/test_sql_refsols/explode_16_postgres.sql b/tests/test_sql_refsols/explode_16_postgres.sql new file mode 100644 index 000000000..1bf61a117 --- /dev/null +++ b/tests/test_sql_refsols/explode_16_postgres.sql @@ -0,0 +1,31 @@ +WITH _q_0 AS ( + SELECT + c_comment AS comment, + c_custkey AS key + FROM tpch.customer + ORDER BY + 2 NULLS FIRST + LIMIT 3 +), _t AS ( + SELECT + _s0.idx - 1 AS idx1, + _s1.idx - 1 AS idx2, + _s2.idx - 1 AS idx3, + _q_0.key, + _s2.val AS val3, + ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.idx - 1, _s1.idx - 1 DESC, _s2.idx - 1) AS _w + FROM _q_0 AS _q_0 + CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) + CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx) + JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) + ON _s2.val <> '' +) +SELECT + key, + idx1, + idx2, + idx3, + val3 +FROM _t +WHERE + _w = 1 diff --git a/tests/test_sql_refsols/explode_16_snowflake.sql b/tests/test_sql_refsols/explode_16_snowflake.sql new file mode 100644 index 000000000..a157b54ae --- /dev/null +++ b/tests/test_sql_refsols/explode_16_snowflake.sql @@ -0,0 +1,30 @@ +WITH _q_0 AS ( + SELECT + c_comment AS comment, + c_custkey AS key + FROM tpch.customer + ORDER BY + 2 NULLS FIRST + LIMIT 3 +), _t0 AS ( + SELECT + _s0.index - 1 AS idx1, + _s1.index - 1 AS idx2, + _s2.index - 1 AS idx3, + _q_0.key, + _s2.value AS val3 + FROM _q_0 AS _q_0 + CROSS JOIN LATERAL SPLIT_TO_TABLE(_q_0.comment, '.') AS _s0 + CROSS JOIN LATERAL SPLIT_TO_TABLE(_s0.value, ',') AS _s1 + JOIN LATERAL SPLIT_TO_TABLE(_s1.value, ' ') AS _s2 + ON _s2.value <> '' + QUALIFY + ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.index - 1, _s1.index - 1 DESC, _s2.index - 1) = 1 +) +SELECT + key, + idx1, + idx2, + idx3, + val3 +FROM _t0 diff --git a/tests/test_sql_refsols/explode_16_trino.sql b/tests/test_sql_refsols/explode_16_trino.sql new file mode 100644 index 000000000..531007657 --- /dev/null +++ b/tests/test_sql_refsols/explode_16_trino.sql @@ -0,0 +1,31 @@ +WITH _q_0 AS ( + SELECT + c_comment AS comment, + c_custkey AS key + FROM tpch.customer + ORDER BY + 2 NULLS FIRST + LIMIT 3 +), _t AS ( + SELECT + _s0.idx - 1 AS idx1, + _s1.idx - 1 AS idx2, + _s2.idx - 1 AS idx3, + _q_0.key, + _s2.val AS val3, + ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.idx - 1, _s1.idx - 1 DESC NULLS FIRST, _s2.idx - 1) AS _w + FROM _q_0 AS _q_0 + CROSS JOIN UNNEST(SPLIT(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) + CROSS JOIN UNNEST(SPLIT(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx) + JOIN UNNEST(SPLIT(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) + ON _s2.val <> '' +) +SELECT + key, + idx1, + idx2, + idx3, + val3 +FROM _t +WHERE + _w = 1 diff --git a/tests/test_sql_refsols/explode_17_databricks.sql b/tests/test_sql_refsols/explode_17_databricks.sql new file mode 100644 index 000000000..ee478723b --- /dev/null +++ b/tests/test_sql_refsols/explode_17_databricks.sql @@ -0,0 +1,30 @@ +WITH _q_0 AS ( + SELECT + c_comment AS comment, + c_custkey AS key + FROM tpch.customer + ORDER BY + 2 + LIMIT 3 +), _t0 AS ( + SELECT + _s0.idx AS idx1, + _s1.idx AS idx2, + _s2.idx AS idx3, + _q_0.key, + _s2.val AS val3 + FROM _q_0 AS _q_0 + CROSS JOIN LATERAL POSEXPLODE(SPLIT(_q_0.comment, '\\Q.\\E')) AS _s0(idx, val) + CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s0.val, '\\Q,\\E')) AS _s1(idx, val) + JOIN LATERAL POSEXPLODE(SPLIT(_s1.val, '\\Q \\E')) AS _s2(idx, val) + ON _s2.val <> '' + QUALIFY + ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.idx DESC NULLS FIRST, _s1.idx DESC NULLS FIRST, _s2.idx NULLS LAST) = 1 +) +SELECT + key, + idx1, + idx2, + idx3, + val3 +FROM _t0 diff --git a/tests/test_sql_refsols/explode_17_duckdb.sql b/tests/test_sql_refsols/explode_17_duckdb.sql new file mode 100644 index 000000000..e25cece0d --- /dev/null +++ b/tests/test_sql_refsols/explode_17_duckdb.sql @@ -0,0 +1,42 @@ +WITH _q_0 AS ( + SELECT + c_comment AS comment, + c_custkey AS key + FROM tpch.customer + ORDER BY + 2 NULLS FIRST + LIMIT 3 +), _t0 AS ( + SELECT + _s0.idx AS idx1, + _s1.idx AS idx2, + _s2.idx AS idx3, + _q_0.key, + _s2.val AS val3 + FROM _q_0 AS _q_0 + CROSS JOIN LATERAL ( + SELECT + UNNEST(STR_SPLIT(_q_0.comment, '.')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_q_0.comment, '.'), 1) - 1 AS _col_1 + ) AS _s0(val, idx) + CROSS JOIN LATERAL ( + SELECT + UNNEST(STR_SPLIT(_s0.val, ',')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_s0.val, ','), 1) - 1 AS _col_1 + ) AS _s1(val, idx) + JOIN LATERAL ( + SELECT + UNNEST(STR_SPLIT(_s1.val, ' ')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_s1.val, ' '), 1) - 1 AS _col_1 + ) AS _s2(val, idx) + ON _s2.val <> '' + QUALIFY + ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.idx DESC NULLS FIRST, _s1.idx DESC NULLS FIRST, _s2.idx) = 1 +) +SELECT + key, + idx1, + idx2, + idx3, + val3 +FROM _t0 diff --git a/tests/test_sql_refsols/explode_17_postgres.sql b/tests/test_sql_refsols/explode_17_postgres.sql new file mode 100644 index 000000000..f8ceda0ce --- /dev/null +++ b/tests/test_sql_refsols/explode_17_postgres.sql @@ -0,0 +1,31 @@ +WITH _q_0 AS ( + SELECT + c_comment AS comment, + c_custkey AS key + FROM tpch.customer + ORDER BY + 2 NULLS FIRST + LIMIT 3 +), _t AS ( + SELECT + _s0.idx - 1 AS idx1, + _s1.idx - 1 AS idx2, + _s2.idx - 1 AS idx3, + _q_0.key, + _s2.val AS val3, + ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.idx - 1 DESC, _s1.idx - 1 DESC, _s2.idx - 1) AS _w + FROM _q_0 AS _q_0 + CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) + CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx) + JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) + ON _s2.val <> '' +) +SELECT + key, + idx1, + idx2, + idx3, + val3 +FROM _t +WHERE + _w = 1 diff --git a/tests/test_sql_refsols/explode_17_snowflake.sql b/tests/test_sql_refsols/explode_17_snowflake.sql new file mode 100644 index 000000000..ce9f9aa23 --- /dev/null +++ b/tests/test_sql_refsols/explode_17_snowflake.sql @@ -0,0 +1,30 @@ +WITH _q_0 AS ( + SELECT + c_comment AS comment, + c_custkey AS key + FROM tpch.customer + ORDER BY + 2 NULLS FIRST + LIMIT 3 +), _t0 AS ( + SELECT + _s0.index - 1 AS idx1, + _s1.index - 1 AS idx2, + _s2.index - 1 AS idx3, + _q_0.key, + _s2.value AS val3 + FROM _q_0 AS _q_0 + CROSS JOIN LATERAL SPLIT_TO_TABLE(_q_0.comment, '.') AS _s0 + CROSS JOIN LATERAL SPLIT_TO_TABLE(_s0.value, ',') AS _s1 + JOIN LATERAL SPLIT_TO_TABLE(_s1.value, ' ') AS _s2 + ON _s2.value <> '' + QUALIFY + ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.index - 1 DESC, _s1.index - 1 DESC, _s2.index - 1) = 1 +) +SELECT + key, + idx1, + idx2, + idx3, + val3 +FROM _t0 diff --git a/tests/test_sql_refsols/explode_17_trino.sql b/tests/test_sql_refsols/explode_17_trino.sql new file mode 100644 index 000000000..b67c75e6d --- /dev/null +++ b/tests/test_sql_refsols/explode_17_trino.sql @@ -0,0 +1,31 @@ +WITH _q_0 AS ( + SELECT + c_comment AS comment, + c_custkey AS key + FROM tpch.customer + ORDER BY + 2 NULLS FIRST + LIMIT 3 +), _t AS ( + SELECT + _s0.idx - 1 AS idx1, + _s1.idx - 1 AS idx2, + _s2.idx - 1 AS idx3, + _q_0.key, + _s2.val AS val3, + ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.idx - 1 DESC NULLS FIRST, _s1.idx - 1 DESC NULLS FIRST, _s2.idx - 1) AS _w + FROM _q_0 AS _q_0 + CROSS JOIN UNNEST(SPLIT(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) + CROSS JOIN UNNEST(SPLIT(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx) + JOIN UNNEST(SPLIT(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) + ON _s2.val <> '' +) +SELECT + key, + idx1, + idx2, + idx3, + val3 +FROM _t +WHERE + _w = 1 diff --git a/tests/test_sql_refsols/explode_18_databricks.sql b/tests/test_sql_refsols/explode_18_databricks.sql new file mode 100644 index 000000000..9ab73832b --- /dev/null +++ b/tests/test_sql_refsols/explode_18_databricks.sql @@ -0,0 +1,30 @@ +WITH _q_0 AS ( + SELECT + c_comment AS comment, + c_custkey AS key + FROM tpch.customer + ORDER BY + 2 + LIMIT 3 +), _t0 AS ( + SELECT + _s0.idx AS idx1, + _s1.idx AS idx2, + _s2.idx AS idx3, + _q_0.key, + _s2.val AS val3 + FROM _q_0 AS _q_0 + CROSS JOIN LATERAL POSEXPLODE(SPLIT(_q_0.comment, '\\Q.\\E')) AS _s0(idx, val) + CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s0.val, '\\Q,\\E')) AS _s1(idx, val) + JOIN LATERAL POSEXPLODE(SPLIT(_s1.val, '\\Q \\E')) AS _s2(idx, val) + ON _s2.val <> '' + QUALIFY + ROW_NUMBER() OVER (PARTITION BY _s0.idx, _q_0.key, _s1.idx ORDER BY _s2.idx DESC NULLS FIRST) = 1 +) +SELECT + key, + idx1, + idx2, + idx3, + val3 +FROM _t0 diff --git a/tests/test_sql_refsols/explode_18_duckdb.sql b/tests/test_sql_refsols/explode_18_duckdb.sql new file mode 100644 index 000000000..606f9e167 --- /dev/null +++ b/tests/test_sql_refsols/explode_18_duckdb.sql @@ -0,0 +1,42 @@ +WITH _q_0 AS ( + SELECT + c_comment AS comment, + c_custkey AS key + FROM tpch.customer + ORDER BY + 2 NULLS FIRST + LIMIT 3 +), _t0 AS ( + SELECT + _s0.idx AS idx1, + _s1.idx AS idx2, + _s2.idx AS idx3, + _q_0.key, + _s2.val AS val3 + FROM _q_0 AS _q_0 + CROSS JOIN LATERAL ( + SELECT + UNNEST(STR_SPLIT(_q_0.comment, '.')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_q_0.comment, '.'), 1) - 1 AS _col_1 + ) AS _s0(val, idx) + CROSS JOIN LATERAL ( + SELECT + UNNEST(STR_SPLIT(_s0.val, ',')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_s0.val, ','), 1) - 1 AS _col_1 + ) AS _s1(val, idx) + JOIN LATERAL ( + SELECT + UNNEST(STR_SPLIT(_s1.val, ' ')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_s1.val, ' '), 1) - 1 AS _col_1 + ) AS _s2(val, idx) + ON _s2.val <> '' + QUALIFY + ROW_NUMBER() OVER (PARTITION BY _s0.idx, _q_0.key, _s1.idx ORDER BY _s2.idx DESC NULLS FIRST) = 1 +) +SELECT + key, + idx1, + idx2, + idx3, + val3 +FROM _t0 diff --git a/tests/test_sql_refsols/explode_18_postgres.sql b/tests/test_sql_refsols/explode_18_postgres.sql new file mode 100644 index 000000000..59ff0d6c9 --- /dev/null +++ b/tests/test_sql_refsols/explode_18_postgres.sql @@ -0,0 +1,31 @@ +WITH _q_0 AS ( + SELECT + c_comment AS comment, + c_custkey AS key + FROM tpch.customer + ORDER BY + 2 NULLS FIRST + LIMIT 3 +), _t AS ( + SELECT + _s0.idx - 1 AS idx1, + _s1.idx - 1 AS idx2, + _s2.idx - 1 AS idx3, + _q_0.key, + _s2.val AS val3, + ROW_NUMBER() OVER (PARTITION BY _s0.idx - 1, _q_0.key, _s1.idx - 1 ORDER BY _s2.idx - 1 DESC) AS _w + FROM _q_0 AS _q_0 + CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) + CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx) + JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) + ON _s2.val <> '' +) +SELECT + key, + idx1, + idx2, + idx3, + val3 +FROM _t +WHERE + _w = 1 diff --git a/tests/test_sql_refsols/explode_18_snowflake.sql b/tests/test_sql_refsols/explode_18_snowflake.sql new file mode 100644 index 000000000..411f34c04 --- /dev/null +++ b/tests/test_sql_refsols/explode_18_snowflake.sql @@ -0,0 +1,30 @@ +WITH _q_0 AS ( + SELECT + c_comment AS comment, + c_custkey AS key + FROM tpch.customer + ORDER BY + 2 NULLS FIRST + LIMIT 3 +), _t0 AS ( + SELECT + _s0.index - 1 AS idx1, + _s1.index - 1 AS idx2, + _s2.index - 1 AS idx3, + _q_0.key, + _s2.value AS val3 + FROM _q_0 AS _q_0 + CROSS JOIN LATERAL SPLIT_TO_TABLE(_q_0.comment, '.') AS _s0 + CROSS JOIN LATERAL SPLIT_TO_TABLE(_s0.value, ',') AS _s1 + JOIN LATERAL SPLIT_TO_TABLE(_s1.value, ' ') AS _s2 + ON _s2.value <> '' + QUALIFY + ROW_NUMBER() OVER (PARTITION BY _s0.index - 1, _q_0.key, _s1.index - 1 ORDER BY _s2.index - 1 DESC) = 1 +) +SELECT + key, + idx1, + idx2, + idx3, + val3 +FROM _t0 diff --git a/tests/test_sql_refsols/explode_18_trino.sql b/tests/test_sql_refsols/explode_18_trino.sql new file mode 100644 index 000000000..26f236b4c --- /dev/null +++ b/tests/test_sql_refsols/explode_18_trino.sql @@ -0,0 +1,31 @@ +WITH _q_0 AS ( + SELECT + c_comment AS comment, + c_custkey AS key + FROM tpch.customer + ORDER BY + 2 NULLS FIRST + LIMIT 3 +), _t AS ( + SELECT + _s0.idx - 1 AS idx1, + _s1.idx - 1 AS idx2, + _s2.idx - 1 AS idx3, + _q_0.key, + _s2.val AS val3, + ROW_NUMBER() OVER (PARTITION BY _s0.idx - 1, _q_0.key, _s1.idx - 1 ORDER BY _s2.idx - 1 DESC NULLS FIRST) AS _w + FROM _q_0 AS _q_0 + CROSS JOIN UNNEST(SPLIT(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) + CROSS JOIN UNNEST(SPLIT(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx) + JOIN UNNEST(SPLIT(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) + ON _s2.val <> '' +) +SELECT + key, + idx1, + idx2, + idx3, + val3 +FROM _t +WHERE + _w = 1 diff --git a/tests/test_sql_refsols/explode_19_databricks.sql b/tests/test_sql_refsols/explode_19_databricks.sql new file mode 100644 index 000000000..d229858f3 --- /dev/null +++ b/tests/test_sql_refsols/explode_19_databricks.sql @@ -0,0 +1,30 @@ +WITH _q_0 AS ( + SELECT + c_comment AS comment, + c_custkey AS key + FROM tpch.customer + ORDER BY + 2 + LIMIT 3 +), _t0 AS ( + SELECT + _s0.idx AS idx1, + _s1.idx AS idx2, + _s2.idx AS idx3, + _q_0.key, + _s2.val AS val3 + FROM _q_0 AS _q_0 + CROSS JOIN LATERAL POSEXPLODE(SPLIT(_q_0.comment, '\\Q.\\E')) AS _s0(idx, val) + CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s0.val, '\\Q,\\E')) AS _s1(idx, val) + JOIN LATERAL POSEXPLODE(SPLIT(_s1.val, '\\Q \\E')) AS _s2(idx, val) + ON _s2.val <> '' + QUALIFY + ROW_NUMBER() OVER (PARTITION BY _q_0.key, _s0.idx ORDER BY _s1.idx DESC NULLS FIRST, _s2.idx DESC NULLS FIRST) = 1 +) +SELECT + key, + idx1, + idx2, + idx3, + val3 +FROM _t0 diff --git a/tests/test_sql_refsols/explode_19_duckdb.sql b/tests/test_sql_refsols/explode_19_duckdb.sql new file mode 100644 index 000000000..f33dd9128 --- /dev/null +++ b/tests/test_sql_refsols/explode_19_duckdb.sql @@ -0,0 +1,42 @@ +WITH _q_0 AS ( + SELECT + c_comment AS comment, + c_custkey AS key + FROM tpch.customer + ORDER BY + 2 NULLS FIRST + LIMIT 3 +), _t0 AS ( + SELECT + _s0.idx AS idx1, + _s1.idx AS idx2, + _s2.idx AS idx3, + _q_0.key, + _s2.val AS val3 + FROM _q_0 AS _q_0 + CROSS JOIN LATERAL ( + SELECT + UNNEST(STR_SPLIT(_q_0.comment, '.')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_q_0.comment, '.'), 1) - 1 AS _col_1 + ) AS _s0(val, idx) + CROSS JOIN LATERAL ( + SELECT + UNNEST(STR_SPLIT(_s0.val, ',')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_s0.val, ','), 1) - 1 AS _col_1 + ) AS _s1(val, idx) + JOIN LATERAL ( + SELECT + UNNEST(STR_SPLIT(_s1.val, ' ')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_s1.val, ' '), 1) - 1 AS _col_1 + ) AS _s2(val, idx) + ON _s2.val <> '' + QUALIFY + ROW_NUMBER() OVER (PARTITION BY _q_0.key, _s0.idx ORDER BY _s1.idx DESC NULLS FIRST, _s2.idx DESC NULLS FIRST) = 1 +) +SELECT + key, + idx1, + idx2, + idx3, + val3 +FROM _t0 diff --git a/tests/test_sql_refsols/explode_19_postgres.sql b/tests/test_sql_refsols/explode_19_postgres.sql new file mode 100644 index 000000000..8aacd9806 --- /dev/null +++ b/tests/test_sql_refsols/explode_19_postgres.sql @@ -0,0 +1,31 @@ +WITH _q_0 AS ( + SELECT + c_comment AS comment, + c_custkey AS key + FROM tpch.customer + ORDER BY + 2 NULLS FIRST + LIMIT 3 +), _t AS ( + SELECT + _s0.idx - 1 AS idx1, + _s1.idx - 1 AS idx2, + _s2.idx - 1 AS idx3, + _q_0.key, + _s2.val AS val3, + ROW_NUMBER() OVER (PARTITION BY _q_0.key, _s0.idx - 1 ORDER BY _s1.idx - 1 DESC, _s2.idx - 1 DESC) AS _w + FROM _q_0 AS _q_0 + CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) + CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx) + JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) + ON _s2.val <> '' +) +SELECT + key, + idx1, + idx2, + idx3, + val3 +FROM _t +WHERE + _w = 1 diff --git a/tests/test_sql_refsols/explode_19_snowflake.sql b/tests/test_sql_refsols/explode_19_snowflake.sql new file mode 100644 index 000000000..d9e2cfffc --- /dev/null +++ b/tests/test_sql_refsols/explode_19_snowflake.sql @@ -0,0 +1,30 @@ +WITH _q_0 AS ( + SELECT + c_comment AS comment, + c_custkey AS key + FROM tpch.customer + ORDER BY + 2 NULLS FIRST + LIMIT 3 +), _t0 AS ( + SELECT + _s0.index - 1 AS idx1, + _s1.index - 1 AS idx2, + _s2.index - 1 AS idx3, + _q_0.key, + _s2.value AS val3 + FROM _q_0 AS _q_0 + CROSS JOIN LATERAL SPLIT_TO_TABLE(_q_0.comment, '.') AS _s0 + CROSS JOIN LATERAL SPLIT_TO_TABLE(_s0.value, ',') AS _s1 + JOIN LATERAL SPLIT_TO_TABLE(_s1.value, ' ') AS _s2 + ON _s2.value <> '' + QUALIFY + ROW_NUMBER() OVER (PARTITION BY _q_0.key, _s0.index - 1 ORDER BY _s1.index - 1 DESC, _s2.index - 1 DESC) = 1 +) +SELECT + key, + idx1, + idx2, + idx3, + val3 +FROM _t0 diff --git a/tests/test_sql_refsols/explode_19_trino.sql b/tests/test_sql_refsols/explode_19_trino.sql new file mode 100644 index 000000000..c5a2957df --- /dev/null +++ b/tests/test_sql_refsols/explode_19_trino.sql @@ -0,0 +1,31 @@ +WITH _q_0 AS ( + SELECT + c_comment AS comment, + c_custkey AS key + FROM tpch.customer + ORDER BY + 2 NULLS FIRST + LIMIT 3 +), _t AS ( + SELECT + _s0.idx - 1 AS idx1, + _s1.idx - 1 AS idx2, + _s2.idx - 1 AS idx3, + _q_0.key, + _s2.val AS val3, + ROW_NUMBER() OVER (PARTITION BY _q_0.key, _s0.idx - 1 ORDER BY _s1.idx - 1 DESC NULLS FIRST, _s2.idx - 1 DESC NULLS FIRST) AS _w + FROM _q_0 AS _q_0 + CROSS JOIN UNNEST(SPLIT(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) + CROSS JOIN UNNEST(SPLIT(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx) + JOIN UNNEST(SPLIT(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) + ON _s2.val <> '' +) +SELECT + key, + idx1, + idx2, + idx3, + val3 +FROM _t +WHERE + _w = 1 diff --git a/tests/test_sql_refsols/explode_20_databricks.sql b/tests/test_sql_refsols/explode_20_databricks.sql new file mode 100644 index 000000000..01f749310 --- /dev/null +++ b/tests/test_sql_refsols/explode_20_databricks.sql @@ -0,0 +1,16 @@ +WITH _q_0 AS ( + SELECT + c_comment AS comment + FROM tpch.customer + ORDER BY + c_custkey + LIMIT 3 +) +SELECT + _s0.val AS char, + COUNT(*) AS n +FROM _q_0 AS _q_0 +JOIN LATERAL POSEXPLODE(SPLIT(_q_0.comment, '\\Q\\E')) AS _s0(idx, val) + ON _s0.val <> '' +GROUP BY + 1 diff --git a/tests/test_sql_refsols/explode_20_duckdb.sql b/tests/test_sql_refsols/explode_20_duckdb.sql new file mode 100644 index 000000000..93d7ca8f3 --- /dev/null +++ b/tests/test_sql_refsols/explode_20_duckdb.sql @@ -0,0 +1,20 @@ +WITH _q_0 AS ( + SELECT + c_comment AS comment + FROM tpch.customer + ORDER BY + c_custkey NULLS FIRST + LIMIT 3 +) +SELECT + _s0.val AS char, + COUNT(*) AS n +FROM _q_0 AS _q_0 +JOIN LATERAL ( + SELECT + UNNEST(REGEXP_SPLIT_TO_ARRAY(_q_0.comment, '')) AS _col_0, + GENERATE_SUBSCRIPTS(REGEXP_SPLIT_TO_ARRAY(_q_0.comment, ''), 1) - 1 AS _col_1 +) AS _s0(val, idx) + ON _s0.val <> '' +GROUP BY + 1 diff --git a/tests/test_sql_refsols/explode_20_postgres.sql b/tests/test_sql_refsols/explode_20_postgres.sql new file mode 100644 index 000000000..d4ebeb10c --- /dev/null +++ b/tests/test_sql_refsols/explode_20_postgres.sql @@ -0,0 +1,16 @@ +WITH _q_0 AS ( + SELECT + c_comment AS comment + FROM tpch.customer + ORDER BY + c_custkey NULLS FIRST + LIMIT 3 +) +SELECT + _s0.val AS char, + COUNT(*) AS n +FROM _q_0 AS _q_0 +JOIN LATERAL UNNEST(REGEXP_SPLIT_TO_ARRAY(_q_0.comment, '')) WITH ORDINALITY AS _s0(val, idx) + ON _s0.val <> '' +GROUP BY + 1 diff --git a/tests/test_sql_refsols/explode_20_snowflake.sql b/tests/test_sql_refsols/explode_20_snowflake.sql new file mode 100644 index 000000000..3099522e4 --- /dev/null +++ b/tests/test_sql_refsols/explode_20_snowflake.sql @@ -0,0 +1,16 @@ +WITH _q_0 AS ( + SELECT + c_comment AS comment + FROM tpch.customer + ORDER BY + c_custkey NULLS FIRST + LIMIT 3 +) +SELECT + _s0.value AS char, + COUNT(*) AS n +FROM _q_0 AS _q_0 +JOIN LATERAL FLATTEN(REGEXP_EXTRACT_ALL(_q_0.comment, '.{1}')) AS _s0(seq, key, path, index, value, this) + ON _s0.value <> '' +GROUP BY + 1 diff --git a/tests/test_sql_refsols/explode_20_trino.sql b/tests/test_sql_refsols/explode_20_trino.sql new file mode 100644 index 000000000..a797b4735 --- /dev/null +++ b/tests/test_sql_refsols/explode_20_trino.sql @@ -0,0 +1,16 @@ +WITH _q_0 AS ( + SELECT + c_comment AS comment + FROM tpch.customer + ORDER BY + c_custkey NULLS FIRST + LIMIT 3 +) +SELECT + _s0.val AS char, + COUNT(*) AS n +FROM _q_0 AS _q_0 +JOIN UNNEST(SPLIT(_q_0.comment, '')) WITH ORDINALITY AS _s0(val, idx) + ON _s0.val <> '' +GROUP BY + 1 diff --git a/tests/test_sql_refsols/explode_21_databricks.sql b/tests/test_sql_refsols/explode_21_databricks.sql new file mode 100644 index 000000000..2e66ba3f4 --- /dev/null +++ b/tests/test_sql_refsols/explode_21_databricks.sql @@ -0,0 +1,33 @@ +WITH _q_0 AS ( + SELECT + c_comment AS comment, + c_custkey AS key + FROM tpch.customer + ORDER BY + 2 + LIMIT 3 +), _t0 AS ( + SELECT + _s0.idx AS idx1, + _s1.idx AS idx2, + _s2.idx AS idx3, + _s3.idx AS idx4, + _q_0.key, + _s3.val AS val4 + FROM _q_0 AS _q_0 + CROSS JOIN LATERAL POSEXPLODE(SPLIT(_q_0.comment, '\\Q.\\E')) AS _s0(idx, val) + CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s0.val, '\\Q,\\E')) AS _s1(idx, val) + CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s1.val, '\\Q \\E')) AS _s2(idx, val) + JOIN LATERAL POSEXPLODE(SPLIT(_s2.val, '\\Q\\E')) AS _s3(idx, val) + ON _s3.val <> '' + QUALIFY + ROW_NUMBER() OVER (PARTITION BY _s1.idx, _s0.idx, _q_0.key, _s2.idx ORDER BY _s3.idx NULLS LAST) = 1 +) +SELECT + key, + idx1, + idx2, + idx3, + idx4, + val4 +FROM _t0 diff --git a/tests/test_sql_refsols/explode_21_duckdb.sql b/tests/test_sql_refsols/explode_21_duckdb.sql new file mode 100644 index 000000000..516e6aee4 --- /dev/null +++ b/tests/test_sql_refsols/explode_21_duckdb.sql @@ -0,0 +1,49 @@ +WITH _q_0 AS ( + SELECT + c_comment AS comment, + c_custkey AS key + FROM tpch.customer + ORDER BY + 2 NULLS FIRST + LIMIT 3 +), _t0 AS ( + SELECT + _s0.idx AS idx1, + _s1.idx AS idx2, + _s2.idx AS idx3, + _s3.idx AS idx4, + _q_0.key, + _s3.val AS val4 + FROM _q_0 AS _q_0 + CROSS JOIN LATERAL ( + SELECT + UNNEST(STR_SPLIT(_q_0.comment, '.')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_q_0.comment, '.'), 1) - 1 AS _col_1 + ) AS _s0(val, idx) + CROSS JOIN LATERAL ( + SELECT + UNNEST(STR_SPLIT(_s0.val, ',')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_s0.val, ','), 1) - 1 AS _col_1 + ) AS _s1(val, idx) + CROSS JOIN LATERAL ( + SELECT + UNNEST(STR_SPLIT(_s1.val, ' ')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_s1.val, ' '), 1) - 1 AS _col_1 + ) AS _s2(val, idx) + JOIN LATERAL ( + SELECT + UNNEST(REGEXP_SPLIT_TO_ARRAY(_s2.val, '')) AS _col_0, + GENERATE_SUBSCRIPTS(REGEXP_SPLIT_TO_ARRAY(_s2.val, ''), 1) - 1 AS _col_1 + ) AS _s3(val, idx) + ON _s3.val <> '' + QUALIFY + ROW_NUMBER() OVER (PARTITION BY _s1.idx, _s0.idx, _q_0.key, _s2.idx ORDER BY _s3.idx) = 1 +) +SELECT + key, + idx1, + idx2, + idx3, + idx4, + val4 +FROM _t0 diff --git a/tests/test_sql_refsols/explode_21_postgres.sql b/tests/test_sql_refsols/explode_21_postgres.sql new file mode 100644 index 000000000..c03272eca --- /dev/null +++ b/tests/test_sql_refsols/explode_21_postgres.sql @@ -0,0 +1,34 @@ +WITH _q_0 AS ( + SELECT + c_comment AS comment, + c_custkey AS key + FROM tpch.customer + ORDER BY + 2 NULLS FIRST + LIMIT 3 +), _t AS ( + SELECT + _s0.idx - 1 AS idx1, + _s1.idx - 1 AS idx2, + _s2.idx - 1 AS idx3, + _s3.idx - 1 AS idx4, + _q_0.key, + _s3.val AS val4, + ROW_NUMBER() OVER (PARTITION BY _s1.idx - 1, _s0.idx - 1, _q_0.key, _s2.idx - 1 ORDER BY _s3.idx - 1) AS _w + FROM _q_0 AS _q_0 + CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) + CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx) + CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) + JOIN LATERAL UNNEST(REGEXP_SPLIT_TO_ARRAY(_s2.val, '')) WITH ORDINALITY AS _s3(val, idx) + ON _s3.val <> '' +) +SELECT + key, + idx1, + idx2, + idx3, + idx4, + val4 +FROM _t +WHERE + _w = 1 diff --git a/tests/test_sql_refsols/explode_21_snowflake.sql b/tests/test_sql_refsols/explode_21_snowflake.sql new file mode 100644 index 000000000..735b578bb --- /dev/null +++ b/tests/test_sql_refsols/explode_21_snowflake.sql @@ -0,0 +1,32 @@ +WITH _q_0 AS ( + SELECT + c_comment AS comment + FROM tpch.customer + ORDER BY + c_custkey NULLS FIRST + LIMIT 3 +), _t0 AS ( + SELECT + _s0.index - 1 AS idx1, + _s1.index - 1 AS idx2, + _s2.index - 1 AS idx3, + _s3.index - 1 AS idx4, + key, + _s3.value AS val4 + FROM _q_0 AS _q_0 + CROSS JOIN LATERAL SPLIT_TO_TABLE(_q_0.comment, '.') AS _s0 + CROSS JOIN LATERAL SPLIT_TO_TABLE(_s0.value, ',') AS _s1 + CROSS JOIN LATERAL SPLIT_TO_TABLE(_s1.value, ' ') AS _s2 + JOIN LATERAL FLATTEN(REGEXP_EXTRACT_ALL(_s2.value, '.{1}')) AS _s3(seq, key, path, index, value, this) + ON _s3.value <> '' + QUALIFY + ROW_NUMBER() OVER (PARTITION BY _s1.index - 1, _s0.index - 1, key, _s2.index - 1 ORDER BY _s3.index - 1) = 1 +) +SELECT + key, + idx1, + idx2, + idx3, + idx4, + val4 +FROM _t0 diff --git a/tests/test_sql_refsols/explode_21_trino.sql b/tests/test_sql_refsols/explode_21_trino.sql new file mode 100644 index 000000000..3c04aecf6 --- /dev/null +++ b/tests/test_sql_refsols/explode_21_trino.sql @@ -0,0 +1,34 @@ +WITH _q_0 AS ( + SELECT + c_comment AS comment, + c_custkey AS key + FROM tpch.customer + ORDER BY + 2 NULLS FIRST + LIMIT 3 +), _t AS ( + SELECT + _s0.idx - 1 AS idx1, + _s1.idx - 1 AS idx2, + _s2.idx - 1 AS idx3, + _s3.idx - 1 AS idx4, + _q_0.key, + _s3.val AS val4, + ROW_NUMBER() OVER (PARTITION BY _s1.idx - 1, _s0.idx - 1, _q_0.key, _s2.idx - 1 ORDER BY _s3.idx - 1) AS _w + FROM _q_0 AS _q_0 + CROSS JOIN UNNEST(SPLIT(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) + CROSS JOIN UNNEST(SPLIT(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx) + CROSS JOIN UNNEST(SPLIT(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) + JOIN UNNEST(SPLIT(_s2.val, '')) WITH ORDINALITY AS _s3(val, idx) + ON _s3.val <> '' +) +SELECT + key, + idx1, + idx2, + idx3, + idx4, + val4 +FROM _t +WHERE + _w = 1 From 4910fc5a25b1b82e83c15ad63b7e9f9532ff580a Mon Sep 17 00:00:00 2001 From: knassre-bodo Date: Mon, 17 Aug 2026 11:08:58 -0700 Subject: [PATCH 30/44] Fixing LATERAL ON bug --- pydough/sqlglot/execute_relational.py | 49 +++++++++++++++++++ tests/test_metadata/bodosql_graphs.json | 13 +---- .../explode_08_databricks.sql | 6 +-- tests/test_sql_refsols/explode_08_duckdb.sql | 6 +-- .../test_sql_refsols/explode_08_postgres.sql | 6 +-- .../test_sql_refsols/explode_08_snowflake.sql | 6 +-- tests/test_sql_refsols/explode_08_trino.sql | 6 +-- .../explode_14_databricks.sql | 6 +-- tests/test_sql_refsols/explode_14_duckdb.sql | 6 +-- .../test_sql_refsols/explode_14_postgres.sql | 6 +-- .../test_sql_refsols/explode_14_snowflake.sql | 6 +-- tests/test_sql_refsols/explode_14_trino.sql | 6 +-- .../explode_15_databricks.sql | 6 +-- tests/test_sql_refsols/explode_15_duckdb.sql | 6 +-- .../test_sql_refsols/explode_15_postgres.sql | 6 +-- .../test_sql_refsols/explode_15_snowflake.sql | 6 +-- tests/test_sql_refsols/explode_15_trino.sql | 6 +-- .../explode_16_databricks.sql | 6 +-- tests/test_sql_refsols/explode_16_duckdb.sql | 6 +-- .../test_sql_refsols/explode_16_postgres.sql | 6 +-- .../test_sql_refsols/explode_16_snowflake.sql | 6 +-- tests/test_sql_refsols/explode_16_trino.sql | 6 +-- .../explode_17_databricks.sql | 6 +-- tests/test_sql_refsols/explode_17_duckdb.sql | 6 +-- .../test_sql_refsols/explode_17_postgres.sql | 6 +-- .../test_sql_refsols/explode_17_snowflake.sql | 6 +-- tests/test_sql_refsols/explode_17_trino.sql | 6 +-- .../explode_18_databricks.sql | 6 +-- tests/test_sql_refsols/explode_18_duckdb.sql | 6 +-- .../test_sql_refsols/explode_18_postgres.sql | 6 +-- .../test_sql_refsols/explode_18_snowflake.sql | 6 +-- tests/test_sql_refsols/explode_18_trino.sql | 6 +-- .../explode_19_databricks.sql | 6 +-- tests/test_sql_refsols/explode_19_duckdb.sql | 6 +-- .../test_sql_refsols/explode_19_postgres.sql | 6 +-- .../test_sql_refsols/explode_19_snowflake.sql | 6 +-- tests/test_sql_refsols/explode_19_trino.sql | 6 +-- .../explode_20_databricks.sql | 6 +-- tests/test_sql_refsols/explode_20_duckdb.sql | 6 +-- .../test_sql_refsols/explode_20_postgres.sql | 6 +-- .../test_sql_refsols/explode_20_snowflake.sql | 6 +-- tests/test_sql_refsols/explode_20_trino.sql | 6 +-- .../explode_21_databricks.sql | 6 +-- tests/test_sql_refsols/explode_21_duckdb.sql | 6 +-- .../test_sql_refsols/explode_21_postgres.sql | 6 +-- .../test_sql_refsols/explode_21_snowflake.sql | 6 +-- tests/test_sql_refsols/explode_21_trino.sql | 6 +-- tests/testing_utilities.py | 2 +- 48 files changed, 186 insertions(+), 148 deletions(-) diff --git a/pydough/sqlglot/execute_relational.py b/pydough/sqlglot/execute_relational.py index 55dee86b2..ee125e700 100644 --- a/pydough/sqlglot/execute_relational.py +++ b/pydough/sqlglot/execute_relational.py @@ -25,9 +25,14 @@ from sqlglot.errors import SqlglotError from sqlglot.expressions import ( Alias, + And, Column, + Join, + Lateral, Select, Table, + Unnest, + Where, With, ) from sqlglot.expressions import Collate as SQLGlotCollate @@ -223,6 +228,9 @@ def apply_sqlglot_optimizer( # Remove table aliases if there is only one Table source in the FROM clause. remove_table_aliases_conditional(glot_expr) + # Pull out ON conditions from a lateral join into a WHERE clause + remove_lateral_on_conditions(glot_expr) + # Remove the Tuple generated around each row in VALUES during the parsing step. # For example `(ROW(i))` becomes `ROW(i)`. The tuple would generate invalid # SQL. @@ -506,6 +514,47 @@ def remove_table_aliases_conditional(expr: SQLGlotExpression) -> None: remove_table_aliases_conditional(item) +def remove_lateral_on_conditions(expr: SQLGlotExpression) -> None: + """ + Pulls out ON conditions from a lateral join into a WHERE clause. This is + necessary because not every dialect supports ON conditions in lateral joins. + """ + # Find every lateral/unnest join that has an ON condition. + if ( + isinstance(expr, Select) + and expr.args.get("joins") is not None + and len(expr.args.get("joins")) > 0 + ): + popped_conditions: list[SQLGlotExpression] = [] + for join in expr.args.get("joins"): + if ( + isinstance(join, Join) + and join.args.get("on", None) is not None + and join.find(Lateral, Unnest) is not None + ): + popped_conditions.append(join.args.pop("on")) + # After the conditions were removed from the ON clause, insert them + # into the WHERE clause. + if len(popped_conditions) > 0: + combined_condition: SQLGlotExpression = popped_conditions[0] + for condition in popped_conditions[1:]: + combined_condition = And(this=combined_condition, expression=condition) + if expr.args.get("where") is None: + expr.set("where", Where(this=combined_condition)) + else: + expr.set( + "where", + Where( + this=And( + this=expr.args.get("where"), expression=combined_condition + ) + ), + ) + # Recursively visit the sub-expressions + for sub_expr in expr.iter_expressions(): + remove_lateral_on_conditions(sub_expr) + + def remove_tuple_row_values(expr: SQLGlotExpression) -> None: """ Visits the AST and removes the tuple if there is only one item and it is diff --git a/tests/test_metadata/bodosql_graphs.json b/tests/test_metadata/bodosql_graphs.json index 932beda9d..6ef0b19bf 100644 --- a/tests/test_metadata/bodosql_graphs.json +++ b/tests/test_metadata/bodosql_graphs.json @@ -260,18 +260,7 @@ "always matches": true, "description": "The company that handled the shipment" } - ], - "functions": [ - { - "name": "LISTOF", - "type": "sql alias", - "aggregation": true, - "sql function": "ARRAY_AGG", - "description": "Aggregates a collection of elements into an array.", - "input signature": {"type": "fixed arguments", "value": ["unknown"]}, - "output signature": {"type": "constant", "value": "array[unknown]"} - } - ], + ] "additional definitions": [], "verified pydough analysis": [], "extra semantic info": {} diff --git a/tests/test_sql_refsols/explode_08_databricks.sql b/tests/test_sql_refsols/explode_08_databricks.sql index ef24083d3..a0dc2a2d8 100644 --- a/tests/test_sql_refsols/explode_08_databricks.sql +++ b/tests/test_sql_refsols/explode_08_databricks.sql @@ -15,9 +15,9 @@ SELECT _s2.val AS val3 FROM _q_0 AS _q_0 CROSS JOIN LATERAL POSEXPLODE(SPLIT(_q_0.comment, '\\Q.\\E')) AS _s0(idx, val) -CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s0.val, '\\Q \\E')) AS _s1(idx, val) -JOIN LATERAL POSEXPLODE(SPLIT(_s1.val, '\\Q,\\E')) AS _s2(idx, val) - ON _s2.val <> '' +CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s0.val, '\\Q \\E')) AS _s1(idx, val), LATERAL POSEXPLODE(SPLIT(_s1.val, '\\Q,\\E')) AS _s2(idx, val) +WHERE + _s2.val <> '' ORDER BY 1, 2, diff --git a/tests/test_sql_refsols/explode_08_duckdb.sql b/tests/test_sql_refsols/explode_08_duckdb.sql index 05c702f39..f8f310282 100644 --- a/tests/test_sql_refsols/explode_08_duckdb.sql +++ b/tests/test_sql_refsols/explode_08_duckdb.sql @@ -23,13 +23,13 @@ CROSS JOIN LATERAL ( SELECT UNNEST(STR_SPLIT(_s0.val, ' ')) AS _col_0, GENERATE_SUBSCRIPTS(STR_SPLIT(_s0.val, ' '), 1) - 1 AS _col_1 -) AS _s1(val, idx) -JOIN LATERAL ( +) AS _s1(val, idx), LATERAL ( SELECT UNNEST(STR_SPLIT(_s1.val, ',')) AS _col_0, GENERATE_SUBSCRIPTS(STR_SPLIT(_s1.val, ','), 1) - 1 AS _col_1 ) AS _s2(val, idx) - ON _s2.val <> '' +WHERE + _s2.val <> '' ORDER BY 1 NULLS FIRST, 2 NULLS FIRST, diff --git a/tests/test_sql_refsols/explode_08_postgres.sql b/tests/test_sql_refsols/explode_08_postgres.sql index 76acc5249..702087490 100644 --- a/tests/test_sql_refsols/explode_08_postgres.sql +++ b/tests/test_sql_refsols/explode_08_postgres.sql @@ -15,9 +15,9 @@ SELECT _s2.val AS val3 FROM _q_0 AS _q_0 CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) -CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s0.val, ' ')) WITH ORDINALITY AS _s1(val, idx) -JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s1.val, ',')) WITH ORDINALITY AS _s2(val, idx) - ON _s2.val <> '' +CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s0.val, ' ')) WITH ORDINALITY AS _s1(val, idx), LATERAL UNNEST(STRING_TO_ARRAY(_s1.val, ',')) WITH ORDINALITY AS _s2(val, idx) +WHERE + _s2.val <> '' ORDER BY 1 NULLS FIRST, 2 NULLS FIRST, diff --git a/tests/test_sql_refsols/explode_08_snowflake.sql b/tests/test_sql_refsols/explode_08_snowflake.sql index 07387100a..d3dcfb595 100644 --- a/tests/test_sql_refsols/explode_08_snowflake.sql +++ b/tests/test_sql_refsols/explode_08_snowflake.sql @@ -15,9 +15,9 @@ SELECT _s2.value AS val3 FROM _q_0 AS _q_0 CROSS JOIN LATERAL SPLIT_TO_TABLE(_q_0.comment, '.') AS _s0 -CROSS JOIN LATERAL SPLIT_TO_TABLE(_s0.value, ' ') AS _s1 -JOIN LATERAL SPLIT_TO_TABLE(_s1.value, ',') AS _s2 - ON _s2.value <> '' +CROSS JOIN LATERAL SPLIT_TO_TABLE(_s0.value, ' ') AS _s1, LATERAL SPLIT_TO_TABLE(_s1.value, ',') AS _s2 +WHERE + _s2.value <> '' ORDER BY 1 NULLS FIRST, 2 NULLS FIRST, diff --git a/tests/test_sql_refsols/explode_08_trino.sql b/tests/test_sql_refsols/explode_08_trino.sql index 5cb8de532..a66d99328 100644 --- a/tests/test_sql_refsols/explode_08_trino.sql +++ b/tests/test_sql_refsols/explode_08_trino.sql @@ -15,9 +15,9 @@ SELECT _s2.val AS val3 FROM _q_0 AS _q_0 CROSS JOIN UNNEST(SPLIT(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) -CROSS JOIN UNNEST(SPLIT(_s0.val, ' ')) WITH ORDINALITY AS _s1(val, idx) -JOIN UNNEST(SPLIT(_s1.val, ',')) WITH ORDINALITY AS _s2(val, idx) - ON _s2.val <> '' +CROSS JOIN UNNEST(SPLIT(_s0.val, ' ')) WITH ORDINALITY AS _s1(val, idx), UNNEST(SPLIT(_s1.val, ',')) WITH ORDINALITY AS _s2(val, idx) +WHERE + _s2.val <> '' ORDER BY 1 NULLS FIRST, 2 NULLS FIRST, diff --git a/tests/test_sql_refsols/explode_14_databricks.sql b/tests/test_sql_refsols/explode_14_databricks.sql index 8fc5e40b0..baafdd793 100644 --- a/tests/test_sql_refsols/explode_14_databricks.sql +++ b/tests/test_sql_refsols/explode_14_databricks.sql @@ -15,9 +15,9 @@ WITH _q_0 AS ( _s2.val AS val3 FROM _q_0 AS _q_0 CROSS JOIN LATERAL POSEXPLODE(SPLIT(_q_0.comment, '\\Q.\\E')) AS _s0(idx, val) - CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s0.val, '\\Q,\\E')) AS _s1(idx, val) - JOIN LATERAL POSEXPLODE(SPLIT(_s1.val, '\\Q \\E')) AS _s2(idx, val) - ON _s2.val <> '' + CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s0.val, '\\Q,\\E')) AS _s1(idx, val), LATERAL POSEXPLODE(SPLIT(_s1.val, '\\Q \\E')) AS _s2(idx, val) + WHERE + _s2.val <> '' QUALIFY ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.idx NULLS LAST, _s1.idx NULLS LAST, _s2.idx NULLS LAST) = 1 ) diff --git a/tests/test_sql_refsols/explode_14_duckdb.sql b/tests/test_sql_refsols/explode_14_duckdb.sql index 3d8b4abd9..fc5f04f62 100644 --- a/tests/test_sql_refsols/explode_14_duckdb.sql +++ b/tests/test_sql_refsols/explode_14_duckdb.sql @@ -23,13 +23,13 @@ WITH _q_0 AS ( SELECT UNNEST(STR_SPLIT(_s0.val, ',')) AS _col_0, GENERATE_SUBSCRIPTS(STR_SPLIT(_s0.val, ','), 1) - 1 AS _col_1 - ) AS _s1(val, idx) - JOIN LATERAL ( + ) AS _s1(val, idx), LATERAL ( SELECT UNNEST(STR_SPLIT(_s1.val, ' ')) AS _col_0, GENERATE_SUBSCRIPTS(STR_SPLIT(_s1.val, ' '), 1) - 1 AS _col_1 ) AS _s2(val, idx) - ON _s2.val <> '' + WHERE + _s2.val <> '' QUALIFY ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.idx, _s1.idx, _s2.idx) = 1 ) diff --git a/tests/test_sql_refsols/explode_14_postgres.sql b/tests/test_sql_refsols/explode_14_postgres.sql index 2c2513ff4..6daa460db 100644 --- a/tests/test_sql_refsols/explode_14_postgres.sql +++ b/tests/test_sql_refsols/explode_14_postgres.sql @@ -16,9 +16,9 @@ WITH _q_0 AS ( ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.idx - 1, _s1.idx - 1, _s2.idx - 1) AS _w FROM _q_0 AS _q_0 CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) - CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx) - JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) - ON _s2.val <> '' + CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx), LATERAL UNNEST(STRING_TO_ARRAY(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) + WHERE + _s2.val <> '' ) SELECT key, diff --git a/tests/test_sql_refsols/explode_14_snowflake.sql b/tests/test_sql_refsols/explode_14_snowflake.sql index 8544d8bb5..325a8a493 100644 --- a/tests/test_sql_refsols/explode_14_snowflake.sql +++ b/tests/test_sql_refsols/explode_14_snowflake.sql @@ -15,9 +15,9 @@ WITH _q_0 AS ( _s2.value AS val3 FROM _q_0 AS _q_0 CROSS JOIN LATERAL SPLIT_TO_TABLE(_q_0.comment, '.') AS _s0 - CROSS JOIN LATERAL SPLIT_TO_TABLE(_s0.value, ',') AS _s1 - JOIN LATERAL SPLIT_TO_TABLE(_s1.value, ' ') AS _s2 - ON _s2.value <> '' + CROSS JOIN LATERAL SPLIT_TO_TABLE(_s0.value, ',') AS _s1, LATERAL SPLIT_TO_TABLE(_s1.value, ' ') AS _s2 + WHERE + _s2.value <> '' QUALIFY ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.index - 1, _s1.index - 1, _s2.index - 1) = 1 ) diff --git a/tests/test_sql_refsols/explode_14_trino.sql b/tests/test_sql_refsols/explode_14_trino.sql index 834168167..03d55684c 100644 --- a/tests/test_sql_refsols/explode_14_trino.sql +++ b/tests/test_sql_refsols/explode_14_trino.sql @@ -16,9 +16,9 @@ WITH _q_0 AS ( ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.idx - 1, _s1.idx - 1, _s2.idx - 1) AS _w FROM _q_0 AS _q_0 CROSS JOIN UNNEST(SPLIT(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) - CROSS JOIN UNNEST(SPLIT(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx) - JOIN UNNEST(SPLIT(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) - ON _s2.val <> '' + CROSS JOIN UNNEST(SPLIT(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx), UNNEST(SPLIT(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) + WHERE + _s2.val <> '' ) SELECT key, diff --git a/tests/test_sql_refsols/explode_15_databricks.sql b/tests/test_sql_refsols/explode_15_databricks.sql index a7e97c983..b84591570 100644 --- a/tests/test_sql_refsols/explode_15_databricks.sql +++ b/tests/test_sql_refsols/explode_15_databricks.sql @@ -15,9 +15,9 @@ WITH _q_0 AS ( _s2.val AS val3 FROM _q_0 AS _q_0 CROSS JOIN LATERAL POSEXPLODE(SPLIT(_q_0.comment, '\\Q.\\E')) AS _s0(idx, val) - CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s0.val, '\\Q,\\E')) AS _s1(idx, val) - JOIN LATERAL POSEXPLODE(SPLIT(_s1.val, '\\Q \\E')) AS _s2(idx, val) - ON _s2.val <> '' + CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s0.val, '\\Q,\\E')) AS _s1(idx, val), LATERAL POSEXPLODE(SPLIT(_s1.val, '\\Q \\E')) AS _s2(idx, val) + WHERE + _s2.val <> '' QUALIFY ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.idx DESC NULLS FIRST, _s1.idx NULLS LAST, _s2.idx NULLS LAST) = 1 ) diff --git a/tests/test_sql_refsols/explode_15_duckdb.sql b/tests/test_sql_refsols/explode_15_duckdb.sql index 6742661c1..73ad5c72b 100644 --- a/tests/test_sql_refsols/explode_15_duckdb.sql +++ b/tests/test_sql_refsols/explode_15_duckdb.sql @@ -23,13 +23,13 @@ WITH _q_0 AS ( SELECT UNNEST(STR_SPLIT(_s0.val, ',')) AS _col_0, GENERATE_SUBSCRIPTS(STR_SPLIT(_s0.val, ','), 1) - 1 AS _col_1 - ) AS _s1(val, idx) - JOIN LATERAL ( + ) AS _s1(val, idx), LATERAL ( SELECT UNNEST(STR_SPLIT(_s1.val, ' ')) AS _col_0, GENERATE_SUBSCRIPTS(STR_SPLIT(_s1.val, ' '), 1) - 1 AS _col_1 ) AS _s2(val, idx) - ON _s2.val <> '' + WHERE + _s2.val <> '' QUALIFY ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.idx DESC NULLS FIRST, _s1.idx, _s2.idx) = 1 ) diff --git a/tests/test_sql_refsols/explode_15_postgres.sql b/tests/test_sql_refsols/explode_15_postgres.sql index 31cf22fd5..663ca15d7 100644 --- a/tests/test_sql_refsols/explode_15_postgres.sql +++ b/tests/test_sql_refsols/explode_15_postgres.sql @@ -16,9 +16,9 @@ WITH _q_0 AS ( ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.idx - 1 DESC, _s1.idx - 1, _s2.idx - 1) AS _w FROM _q_0 AS _q_0 CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) - CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx) - JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) - ON _s2.val <> '' + CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx), LATERAL UNNEST(STRING_TO_ARRAY(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) + WHERE + _s2.val <> '' ) SELECT key, diff --git a/tests/test_sql_refsols/explode_15_snowflake.sql b/tests/test_sql_refsols/explode_15_snowflake.sql index f4dd6145f..aa51f1fff 100644 --- a/tests/test_sql_refsols/explode_15_snowflake.sql +++ b/tests/test_sql_refsols/explode_15_snowflake.sql @@ -15,9 +15,9 @@ WITH _q_0 AS ( _s2.value AS val3 FROM _q_0 AS _q_0 CROSS JOIN LATERAL SPLIT_TO_TABLE(_q_0.comment, '.') AS _s0 - CROSS JOIN LATERAL SPLIT_TO_TABLE(_s0.value, ',') AS _s1 - JOIN LATERAL SPLIT_TO_TABLE(_s1.value, ' ') AS _s2 - ON _s2.value <> '' + CROSS JOIN LATERAL SPLIT_TO_TABLE(_s0.value, ',') AS _s1, LATERAL SPLIT_TO_TABLE(_s1.value, ' ') AS _s2 + WHERE + _s2.value <> '' QUALIFY ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.index - 1 DESC, _s1.index - 1, _s2.index - 1) = 1 ) diff --git a/tests/test_sql_refsols/explode_15_trino.sql b/tests/test_sql_refsols/explode_15_trino.sql index f59f5839a..e5767fb89 100644 --- a/tests/test_sql_refsols/explode_15_trino.sql +++ b/tests/test_sql_refsols/explode_15_trino.sql @@ -16,9 +16,9 @@ WITH _q_0 AS ( ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.idx - 1 DESC NULLS FIRST, _s1.idx - 1, _s2.idx - 1) AS _w FROM _q_0 AS _q_0 CROSS JOIN UNNEST(SPLIT(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) - CROSS JOIN UNNEST(SPLIT(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx) - JOIN UNNEST(SPLIT(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) - ON _s2.val <> '' + CROSS JOIN UNNEST(SPLIT(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx), UNNEST(SPLIT(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) + WHERE + _s2.val <> '' ) SELECT key, diff --git a/tests/test_sql_refsols/explode_16_databricks.sql b/tests/test_sql_refsols/explode_16_databricks.sql index 8f2fa19c1..07d9380af 100644 --- a/tests/test_sql_refsols/explode_16_databricks.sql +++ b/tests/test_sql_refsols/explode_16_databricks.sql @@ -15,9 +15,9 @@ WITH _q_0 AS ( _s2.val AS val3 FROM _q_0 AS _q_0 CROSS JOIN LATERAL POSEXPLODE(SPLIT(_q_0.comment, '\\Q.\\E')) AS _s0(idx, val) - CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s0.val, '\\Q,\\E')) AS _s1(idx, val) - JOIN LATERAL POSEXPLODE(SPLIT(_s1.val, '\\Q \\E')) AS _s2(idx, val) - ON _s2.val <> '' + CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s0.val, '\\Q,\\E')) AS _s1(idx, val), LATERAL POSEXPLODE(SPLIT(_s1.val, '\\Q \\E')) AS _s2(idx, val) + WHERE + _s2.val <> '' QUALIFY ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.idx NULLS LAST, _s1.idx DESC NULLS FIRST, _s2.idx NULLS LAST) = 1 ) diff --git a/tests/test_sql_refsols/explode_16_duckdb.sql b/tests/test_sql_refsols/explode_16_duckdb.sql index 11e2bdcfc..2f5d6ce5d 100644 --- a/tests/test_sql_refsols/explode_16_duckdb.sql +++ b/tests/test_sql_refsols/explode_16_duckdb.sql @@ -23,13 +23,13 @@ WITH _q_0 AS ( SELECT UNNEST(STR_SPLIT(_s0.val, ',')) AS _col_0, GENERATE_SUBSCRIPTS(STR_SPLIT(_s0.val, ','), 1) - 1 AS _col_1 - ) AS _s1(val, idx) - JOIN LATERAL ( + ) AS _s1(val, idx), LATERAL ( SELECT UNNEST(STR_SPLIT(_s1.val, ' ')) AS _col_0, GENERATE_SUBSCRIPTS(STR_SPLIT(_s1.val, ' '), 1) - 1 AS _col_1 ) AS _s2(val, idx) - ON _s2.val <> '' + WHERE + _s2.val <> '' QUALIFY ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.idx, _s1.idx DESC NULLS FIRST, _s2.idx) = 1 ) diff --git a/tests/test_sql_refsols/explode_16_postgres.sql b/tests/test_sql_refsols/explode_16_postgres.sql index 1bf61a117..171e508fe 100644 --- a/tests/test_sql_refsols/explode_16_postgres.sql +++ b/tests/test_sql_refsols/explode_16_postgres.sql @@ -16,9 +16,9 @@ WITH _q_0 AS ( ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.idx - 1, _s1.idx - 1 DESC, _s2.idx - 1) AS _w FROM _q_0 AS _q_0 CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) - CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx) - JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) - ON _s2.val <> '' + CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx), LATERAL UNNEST(STRING_TO_ARRAY(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) + WHERE + _s2.val <> '' ) SELECT key, diff --git a/tests/test_sql_refsols/explode_16_snowflake.sql b/tests/test_sql_refsols/explode_16_snowflake.sql index a157b54ae..410f28645 100644 --- a/tests/test_sql_refsols/explode_16_snowflake.sql +++ b/tests/test_sql_refsols/explode_16_snowflake.sql @@ -15,9 +15,9 @@ WITH _q_0 AS ( _s2.value AS val3 FROM _q_0 AS _q_0 CROSS JOIN LATERAL SPLIT_TO_TABLE(_q_0.comment, '.') AS _s0 - CROSS JOIN LATERAL SPLIT_TO_TABLE(_s0.value, ',') AS _s1 - JOIN LATERAL SPLIT_TO_TABLE(_s1.value, ' ') AS _s2 - ON _s2.value <> '' + CROSS JOIN LATERAL SPLIT_TO_TABLE(_s0.value, ',') AS _s1, LATERAL SPLIT_TO_TABLE(_s1.value, ' ') AS _s2 + WHERE + _s2.value <> '' QUALIFY ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.index - 1, _s1.index - 1 DESC, _s2.index - 1) = 1 ) diff --git a/tests/test_sql_refsols/explode_16_trino.sql b/tests/test_sql_refsols/explode_16_trino.sql index 531007657..abc405f1a 100644 --- a/tests/test_sql_refsols/explode_16_trino.sql +++ b/tests/test_sql_refsols/explode_16_trino.sql @@ -16,9 +16,9 @@ WITH _q_0 AS ( ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.idx - 1, _s1.idx - 1 DESC NULLS FIRST, _s2.idx - 1) AS _w FROM _q_0 AS _q_0 CROSS JOIN UNNEST(SPLIT(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) - CROSS JOIN UNNEST(SPLIT(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx) - JOIN UNNEST(SPLIT(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) - ON _s2.val <> '' + CROSS JOIN UNNEST(SPLIT(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx), UNNEST(SPLIT(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) + WHERE + _s2.val <> '' ) SELECT key, diff --git a/tests/test_sql_refsols/explode_17_databricks.sql b/tests/test_sql_refsols/explode_17_databricks.sql index ee478723b..b49c2afaf 100644 --- a/tests/test_sql_refsols/explode_17_databricks.sql +++ b/tests/test_sql_refsols/explode_17_databricks.sql @@ -15,9 +15,9 @@ WITH _q_0 AS ( _s2.val AS val3 FROM _q_0 AS _q_0 CROSS JOIN LATERAL POSEXPLODE(SPLIT(_q_0.comment, '\\Q.\\E')) AS _s0(idx, val) - CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s0.val, '\\Q,\\E')) AS _s1(idx, val) - JOIN LATERAL POSEXPLODE(SPLIT(_s1.val, '\\Q \\E')) AS _s2(idx, val) - ON _s2.val <> '' + CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s0.val, '\\Q,\\E')) AS _s1(idx, val), LATERAL POSEXPLODE(SPLIT(_s1.val, '\\Q \\E')) AS _s2(idx, val) + WHERE + _s2.val <> '' QUALIFY ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.idx DESC NULLS FIRST, _s1.idx DESC NULLS FIRST, _s2.idx NULLS LAST) = 1 ) diff --git a/tests/test_sql_refsols/explode_17_duckdb.sql b/tests/test_sql_refsols/explode_17_duckdb.sql index e25cece0d..d3c268319 100644 --- a/tests/test_sql_refsols/explode_17_duckdb.sql +++ b/tests/test_sql_refsols/explode_17_duckdb.sql @@ -23,13 +23,13 @@ WITH _q_0 AS ( SELECT UNNEST(STR_SPLIT(_s0.val, ',')) AS _col_0, GENERATE_SUBSCRIPTS(STR_SPLIT(_s0.val, ','), 1) - 1 AS _col_1 - ) AS _s1(val, idx) - JOIN LATERAL ( + ) AS _s1(val, idx), LATERAL ( SELECT UNNEST(STR_SPLIT(_s1.val, ' ')) AS _col_0, GENERATE_SUBSCRIPTS(STR_SPLIT(_s1.val, ' '), 1) - 1 AS _col_1 ) AS _s2(val, idx) - ON _s2.val <> '' + WHERE + _s2.val <> '' QUALIFY ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.idx DESC NULLS FIRST, _s1.idx DESC NULLS FIRST, _s2.idx) = 1 ) diff --git a/tests/test_sql_refsols/explode_17_postgres.sql b/tests/test_sql_refsols/explode_17_postgres.sql index f8ceda0ce..01c4d2c92 100644 --- a/tests/test_sql_refsols/explode_17_postgres.sql +++ b/tests/test_sql_refsols/explode_17_postgres.sql @@ -16,9 +16,9 @@ WITH _q_0 AS ( ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.idx - 1 DESC, _s1.idx - 1 DESC, _s2.idx - 1) AS _w FROM _q_0 AS _q_0 CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) - CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx) - JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) - ON _s2.val <> '' + CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx), LATERAL UNNEST(STRING_TO_ARRAY(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) + WHERE + _s2.val <> '' ) SELECT key, diff --git a/tests/test_sql_refsols/explode_17_snowflake.sql b/tests/test_sql_refsols/explode_17_snowflake.sql index ce9f9aa23..623acd389 100644 --- a/tests/test_sql_refsols/explode_17_snowflake.sql +++ b/tests/test_sql_refsols/explode_17_snowflake.sql @@ -15,9 +15,9 @@ WITH _q_0 AS ( _s2.value AS val3 FROM _q_0 AS _q_0 CROSS JOIN LATERAL SPLIT_TO_TABLE(_q_0.comment, '.') AS _s0 - CROSS JOIN LATERAL SPLIT_TO_TABLE(_s0.value, ',') AS _s1 - JOIN LATERAL SPLIT_TO_TABLE(_s1.value, ' ') AS _s2 - ON _s2.value <> '' + CROSS JOIN LATERAL SPLIT_TO_TABLE(_s0.value, ',') AS _s1, LATERAL SPLIT_TO_TABLE(_s1.value, ' ') AS _s2 + WHERE + _s2.value <> '' QUALIFY ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.index - 1 DESC, _s1.index - 1 DESC, _s2.index - 1) = 1 ) diff --git a/tests/test_sql_refsols/explode_17_trino.sql b/tests/test_sql_refsols/explode_17_trino.sql index b67c75e6d..83e26edb0 100644 --- a/tests/test_sql_refsols/explode_17_trino.sql +++ b/tests/test_sql_refsols/explode_17_trino.sql @@ -16,9 +16,9 @@ WITH _q_0 AS ( ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.idx - 1 DESC NULLS FIRST, _s1.idx - 1 DESC NULLS FIRST, _s2.idx - 1) AS _w FROM _q_0 AS _q_0 CROSS JOIN UNNEST(SPLIT(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) - CROSS JOIN UNNEST(SPLIT(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx) - JOIN UNNEST(SPLIT(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) - ON _s2.val <> '' + CROSS JOIN UNNEST(SPLIT(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx), UNNEST(SPLIT(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) + WHERE + _s2.val <> '' ) SELECT key, diff --git a/tests/test_sql_refsols/explode_18_databricks.sql b/tests/test_sql_refsols/explode_18_databricks.sql index 9ab73832b..700c8264d 100644 --- a/tests/test_sql_refsols/explode_18_databricks.sql +++ b/tests/test_sql_refsols/explode_18_databricks.sql @@ -15,9 +15,9 @@ WITH _q_0 AS ( _s2.val AS val3 FROM _q_0 AS _q_0 CROSS JOIN LATERAL POSEXPLODE(SPLIT(_q_0.comment, '\\Q.\\E')) AS _s0(idx, val) - CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s0.val, '\\Q,\\E')) AS _s1(idx, val) - JOIN LATERAL POSEXPLODE(SPLIT(_s1.val, '\\Q \\E')) AS _s2(idx, val) - ON _s2.val <> '' + CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s0.val, '\\Q,\\E')) AS _s1(idx, val), LATERAL POSEXPLODE(SPLIT(_s1.val, '\\Q \\E')) AS _s2(idx, val) + WHERE + _s2.val <> '' QUALIFY ROW_NUMBER() OVER (PARTITION BY _s0.idx, _q_0.key, _s1.idx ORDER BY _s2.idx DESC NULLS FIRST) = 1 ) diff --git a/tests/test_sql_refsols/explode_18_duckdb.sql b/tests/test_sql_refsols/explode_18_duckdb.sql index 606f9e167..582ae1ca5 100644 --- a/tests/test_sql_refsols/explode_18_duckdb.sql +++ b/tests/test_sql_refsols/explode_18_duckdb.sql @@ -23,13 +23,13 @@ WITH _q_0 AS ( SELECT UNNEST(STR_SPLIT(_s0.val, ',')) AS _col_0, GENERATE_SUBSCRIPTS(STR_SPLIT(_s0.val, ','), 1) - 1 AS _col_1 - ) AS _s1(val, idx) - JOIN LATERAL ( + ) AS _s1(val, idx), LATERAL ( SELECT UNNEST(STR_SPLIT(_s1.val, ' ')) AS _col_0, GENERATE_SUBSCRIPTS(STR_SPLIT(_s1.val, ' '), 1) - 1 AS _col_1 ) AS _s2(val, idx) - ON _s2.val <> '' + WHERE + _s2.val <> '' QUALIFY ROW_NUMBER() OVER (PARTITION BY _s0.idx, _q_0.key, _s1.idx ORDER BY _s2.idx DESC NULLS FIRST) = 1 ) diff --git a/tests/test_sql_refsols/explode_18_postgres.sql b/tests/test_sql_refsols/explode_18_postgres.sql index 59ff0d6c9..a6e221595 100644 --- a/tests/test_sql_refsols/explode_18_postgres.sql +++ b/tests/test_sql_refsols/explode_18_postgres.sql @@ -16,9 +16,9 @@ WITH _q_0 AS ( ROW_NUMBER() OVER (PARTITION BY _s0.idx - 1, _q_0.key, _s1.idx - 1 ORDER BY _s2.idx - 1 DESC) AS _w FROM _q_0 AS _q_0 CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) - CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx) - JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) - ON _s2.val <> '' + CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx), LATERAL UNNEST(STRING_TO_ARRAY(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) + WHERE + _s2.val <> '' ) SELECT key, diff --git a/tests/test_sql_refsols/explode_18_snowflake.sql b/tests/test_sql_refsols/explode_18_snowflake.sql index 411f34c04..e652b0fa7 100644 --- a/tests/test_sql_refsols/explode_18_snowflake.sql +++ b/tests/test_sql_refsols/explode_18_snowflake.sql @@ -15,9 +15,9 @@ WITH _q_0 AS ( _s2.value AS val3 FROM _q_0 AS _q_0 CROSS JOIN LATERAL SPLIT_TO_TABLE(_q_0.comment, '.') AS _s0 - CROSS JOIN LATERAL SPLIT_TO_TABLE(_s0.value, ',') AS _s1 - JOIN LATERAL SPLIT_TO_TABLE(_s1.value, ' ') AS _s2 - ON _s2.value <> '' + CROSS JOIN LATERAL SPLIT_TO_TABLE(_s0.value, ',') AS _s1, LATERAL SPLIT_TO_TABLE(_s1.value, ' ') AS _s2 + WHERE + _s2.value <> '' QUALIFY ROW_NUMBER() OVER (PARTITION BY _s0.index - 1, _q_0.key, _s1.index - 1 ORDER BY _s2.index - 1 DESC) = 1 ) diff --git a/tests/test_sql_refsols/explode_18_trino.sql b/tests/test_sql_refsols/explode_18_trino.sql index 26f236b4c..8a89f472f 100644 --- a/tests/test_sql_refsols/explode_18_trino.sql +++ b/tests/test_sql_refsols/explode_18_trino.sql @@ -16,9 +16,9 @@ WITH _q_0 AS ( ROW_NUMBER() OVER (PARTITION BY _s0.idx - 1, _q_0.key, _s1.idx - 1 ORDER BY _s2.idx - 1 DESC NULLS FIRST) AS _w FROM _q_0 AS _q_0 CROSS JOIN UNNEST(SPLIT(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) - CROSS JOIN UNNEST(SPLIT(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx) - JOIN UNNEST(SPLIT(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) - ON _s2.val <> '' + CROSS JOIN UNNEST(SPLIT(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx), UNNEST(SPLIT(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) + WHERE + _s2.val <> '' ) SELECT key, diff --git a/tests/test_sql_refsols/explode_19_databricks.sql b/tests/test_sql_refsols/explode_19_databricks.sql index d229858f3..fccdda8e9 100644 --- a/tests/test_sql_refsols/explode_19_databricks.sql +++ b/tests/test_sql_refsols/explode_19_databricks.sql @@ -15,9 +15,9 @@ WITH _q_0 AS ( _s2.val AS val3 FROM _q_0 AS _q_0 CROSS JOIN LATERAL POSEXPLODE(SPLIT(_q_0.comment, '\\Q.\\E')) AS _s0(idx, val) - CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s0.val, '\\Q,\\E')) AS _s1(idx, val) - JOIN LATERAL POSEXPLODE(SPLIT(_s1.val, '\\Q \\E')) AS _s2(idx, val) - ON _s2.val <> '' + CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s0.val, '\\Q,\\E')) AS _s1(idx, val), LATERAL POSEXPLODE(SPLIT(_s1.val, '\\Q \\E')) AS _s2(idx, val) + WHERE + _s2.val <> '' QUALIFY ROW_NUMBER() OVER (PARTITION BY _q_0.key, _s0.idx ORDER BY _s1.idx DESC NULLS FIRST, _s2.idx DESC NULLS FIRST) = 1 ) diff --git a/tests/test_sql_refsols/explode_19_duckdb.sql b/tests/test_sql_refsols/explode_19_duckdb.sql index f33dd9128..701e3b0b3 100644 --- a/tests/test_sql_refsols/explode_19_duckdb.sql +++ b/tests/test_sql_refsols/explode_19_duckdb.sql @@ -23,13 +23,13 @@ WITH _q_0 AS ( SELECT UNNEST(STR_SPLIT(_s0.val, ',')) AS _col_0, GENERATE_SUBSCRIPTS(STR_SPLIT(_s0.val, ','), 1) - 1 AS _col_1 - ) AS _s1(val, idx) - JOIN LATERAL ( + ) AS _s1(val, idx), LATERAL ( SELECT UNNEST(STR_SPLIT(_s1.val, ' ')) AS _col_0, GENERATE_SUBSCRIPTS(STR_SPLIT(_s1.val, ' '), 1) - 1 AS _col_1 ) AS _s2(val, idx) - ON _s2.val <> '' + WHERE + _s2.val <> '' QUALIFY ROW_NUMBER() OVER (PARTITION BY _q_0.key, _s0.idx ORDER BY _s1.idx DESC NULLS FIRST, _s2.idx DESC NULLS FIRST) = 1 ) diff --git a/tests/test_sql_refsols/explode_19_postgres.sql b/tests/test_sql_refsols/explode_19_postgres.sql index 8aacd9806..124ff40ff 100644 --- a/tests/test_sql_refsols/explode_19_postgres.sql +++ b/tests/test_sql_refsols/explode_19_postgres.sql @@ -16,9 +16,9 @@ WITH _q_0 AS ( ROW_NUMBER() OVER (PARTITION BY _q_0.key, _s0.idx - 1 ORDER BY _s1.idx - 1 DESC, _s2.idx - 1 DESC) AS _w FROM _q_0 AS _q_0 CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) - CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx) - JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) - ON _s2.val <> '' + CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx), LATERAL UNNEST(STRING_TO_ARRAY(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) + WHERE + _s2.val <> '' ) SELECT key, diff --git a/tests/test_sql_refsols/explode_19_snowflake.sql b/tests/test_sql_refsols/explode_19_snowflake.sql index d9e2cfffc..4d19779f9 100644 --- a/tests/test_sql_refsols/explode_19_snowflake.sql +++ b/tests/test_sql_refsols/explode_19_snowflake.sql @@ -15,9 +15,9 @@ WITH _q_0 AS ( _s2.value AS val3 FROM _q_0 AS _q_0 CROSS JOIN LATERAL SPLIT_TO_TABLE(_q_0.comment, '.') AS _s0 - CROSS JOIN LATERAL SPLIT_TO_TABLE(_s0.value, ',') AS _s1 - JOIN LATERAL SPLIT_TO_TABLE(_s1.value, ' ') AS _s2 - ON _s2.value <> '' + CROSS JOIN LATERAL SPLIT_TO_TABLE(_s0.value, ',') AS _s1, LATERAL SPLIT_TO_TABLE(_s1.value, ' ') AS _s2 + WHERE + _s2.value <> '' QUALIFY ROW_NUMBER() OVER (PARTITION BY _q_0.key, _s0.index - 1 ORDER BY _s1.index - 1 DESC, _s2.index - 1 DESC) = 1 ) diff --git a/tests/test_sql_refsols/explode_19_trino.sql b/tests/test_sql_refsols/explode_19_trino.sql index c5a2957df..c7c2e0420 100644 --- a/tests/test_sql_refsols/explode_19_trino.sql +++ b/tests/test_sql_refsols/explode_19_trino.sql @@ -16,9 +16,9 @@ WITH _q_0 AS ( ROW_NUMBER() OVER (PARTITION BY _q_0.key, _s0.idx - 1 ORDER BY _s1.idx - 1 DESC NULLS FIRST, _s2.idx - 1 DESC NULLS FIRST) AS _w FROM _q_0 AS _q_0 CROSS JOIN UNNEST(SPLIT(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) - CROSS JOIN UNNEST(SPLIT(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx) - JOIN UNNEST(SPLIT(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) - ON _s2.val <> '' + CROSS JOIN UNNEST(SPLIT(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx), UNNEST(SPLIT(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) + WHERE + _s2.val <> '' ) SELECT key, diff --git a/tests/test_sql_refsols/explode_20_databricks.sql b/tests/test_sql_refsols/explode_20_databricks.sql index 01f749310..6ff5c740b 100644 --- a/tests/test_sql_refsols/explode_20_databricks.sql +++ b/tests/test_sql_refsols/explode_20_databricks.sql @@ -9,8 +9,8 @@ WITH _q_0 AS ( SELECT _s0.val AS char, COUNT(*) AS n -FROM _q_0 AS _q_0 -JOIN LATERAL POSEXPLODE(SPLIT(_q_0.comment, '\\Q\\E')) AS _s0(idx, val) - ON _s0.val <> '' +FROM _q_0 AS _q_0, LATERAL POSEXPLODE(SPLIT(_q_0.comment, '\\Q\\E')) AS _s0(idx, val) +WHERE + _s0.val <> '' GROUP BY 1 diff --git a/tests/test_sql_refsols/explode_20_duckdb.sql b/tests/test_sql_refsols/explode_20_duckdb.sql index 93d7ca8f3..6c3da3cff 100644 --- a/tests/test_sql_refsols/explode_20_duckdb.sql +++ b/tests/test_sql_refsols/explode_20_duckdb.sql @@ -9,12 +9,12 @@ WITH _q_0 AS ( SELECT _s0.val AS char, COUNT(*) AS n -FROM _q_0 AS _q_0 -JOIN LATERAL ( +FROM _q_0 AS _q_0, LATERAL ( SELECT UNNEST(REGEXP_SPLIT_TO_ARRAY(_q_0.comment, '')) AS _col_0, GENERATE_SUBSCRIPTS(REGEXP_SPLIT_TO_ARRAY(_q_0.comment, ''), 1) - 1 AS _col_1 ) AS _s0(val, idx) - ON _s0.val <> '' +WHERE + _s0.val <> '' GROUP BY 1 diff --git a/tests/test_sql_refsols/explode_20_postgres.sql b/tests/test_sql_refsols/explode_20_postgres.sql index d4ebeb10c..2320cea42 100644 --- a/tests/test_sql_refsols/explode_20_postgres.sql +++ b/tests/test_sql_refsols/explode_20_postgres.sql @@ -9,8 +9,8 @@ WITH _q_0 AS ( SELECT _s0.val AS char, COUNT(*) AS n -FROM _q_0 AS _q_0 -JOIN LATERAL UNNEST(REGEXP_SPLIT_TO_ARRAY(_q_0.comment, '')) WITH ORDINALITY AS _s0(val, idx) - ON _s0.val <> '' +FROM _q_0 AS _q_0, LATERAL UNNEST(REGEXP_SPLIT_TO_ARRAY(_q_0.comment, '')) WITH ORDINALITY AS _s0(val, idx) +WHERE + _s0.val <> '' GROUP BY 1 diff --git a/tests/test_sql_refsols/explode_20_snowflake.sql b/tests/test_sql_refsols/explode_20_snowflake.sql index 3099522e4..494f067ef 100644 --- a/tests/test_sql_refsols/explode_20_snowflake.sql +++ b/tests/test_sql_refsols/explode_20_snowflake.sql @@ -9,8 +9,8 @@ WITH _q_0 AS ( SELECT _s0.value AS char, COUNT(*) AS n -FROM _q_0 AS _q_0 -JOIN LATERAL FLATTEN(REGEXP_EXTRACT_ALL(_q_0.comment, '.{1}')) AS _s0(seq, key, path, index, value, this) - ON _s0.value <> '' +FROM _q_0 AS _q_0, LATERAL FLATTEN(REGEXP_EXTRACT_ALL(_q_0.comment, '.{1}')) AS _s0(seq, key, path, index, value, this) +WHERE + _s0.value <> '' GROUP BY 1 diff --git a/tests/test_sql_refsols/explode_20_trino.sql b/tests/test_sql_refsols/explode_20_trino.sql index a797b4735..a2363f843 100644 --- a/tests/test_sql_refsols/explode_20_trino.sql +++ b/tests/test_sql_refsols/explode_20_trino.sql @@ -9,8 +9,8 @@ WITH _q_0 AS ( SELECT _s0.val AS char, COUNT(*) AS n -FROM _q_0 AS _q_0 -JOIN UNNEST(SPLIT(_q_0.comment, '')) WITH ORDINALITY AS _s0(val, idx) - ON _s0.val <> '' +FROM _q_0 AS _q_0, UNNEST(SPLIT(_q_0.comment, '')) WITH ORDINALITY AS _s0(val, idx) +WHERE + _s0.val <> '' GROUP BY 1 diff --git a/tests/test_sql_refsols/explode_21_databricks.sql b/tests/test_sql_refsols/explode_21_databricks.sql index 2e66ba3f4..9115278ac 100644 --- a/tests/test_sql_refsols/explode_21_databricks.sql +++ b/tests/test_sql_refsols/explode_21_databricks.sql @@ -17,9 +17,9 @@ WITH _q_0 AS ( FROM _q_0 AS _q_0 CROSS JOIN LATERAL POSEXPLODE(SPLIT(_q_0.comment, '\\Q.\\E')) AS _s0(idx, val) CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s0.val, '\\Q,\\E')) AS _s1(idx, val) - CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s1.val, '\\Q \\E')) AS _s2(idx, val) - JOIN LATERAL POSEXPLODE(SPLIT(_s2.val, '\\Q\\E')) AS _s3(idx, val) - ON _s3.val <> '' + CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s1.val, '\\Q \\E')) AS _s2(idx, val), LATERAL POSEXPLODE(SPLIT(_s2.val, '\\Q\\E')) AS _s3(idx, val) + WHERE + _s3.val <> '' QUALIFY ROW_NUMBER() OVER (PARTITION BY _s1.idx, _s0.idx, _q_0.key, _s2.idx ORDER BY _s3.idx NULLS LAST) = 1 ) diff --git a/tests/test_sql_refsols/explode_21_duckdb.sql b/tests/test_sql_refsols/explode_21_duckdb.sql index 516e6aee4..12f5d4ea6 100644 --- a/tests/test_sql_refsols/explode_21_duckdb.sql +++ b/tests/test_sql_refsols/explode_21_duckdb.sql @@ -29,13 +29,13 @@ WITH _q_0 AS ( SELECT UNNEST(STR_SPLIT(_s1.val, ' ')) AS _col_0, GENERATE_SUBSCRIPTS(STR_SPLIT(_s1.val, ' '), 1) - 1 AS _col_1 - ) AS _s2(val, idx) - JOIN LATERAL ( + ) AS _s2(val, idx), LATERAL ( SELECT UNNEST(REGEXP_SPLIT_TO_ARRAY(_s2.val, '')) AS _col_0, GENERATE_SUBSCRIPTS(REGEXP_SPLIT_TO_ARRAY(_s2.val, ''), 1) - 1 AS _col_1 ) AS _s3(val, idx) - ON _s3.val <> '' + WHERE + _s3.val <> '' QUALIFY ROW_NUMBER() OVER (PARTITION BY _s1.idx, _s0.idx, _q_0.key, _s2.idx ORDER BY _s3.idx) = 1 ) diff --git a/tests/test_sql_refsols/explode_21_postgres.sql b/tests/test_sql_refsols/explode_21_postgres.sql index c03272eca..d36d95c51 100644 --- a/tests/test_sql_refsols/explode_21_postgres.sql +++ b/tests/test_sql_refsols/explode_21_postgres.sql @@ -18,9 +18,9 @@ WITH _q_0 AS ( FROM _q_0 AS _q_0 CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx) - CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) - JOIN LATERAL UNNEST(REGEXP_SPLIT_TO_ARRAY(_s2.val, '')) WITH ORDINALITY AS _s3(val, idx) - ON _s3.val <> '' + CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx), LATERAL UNNEST(REGEXP_SPLIT_TO_ARRAY(_s2.val, '')) WITH ORDINALITY AS _s3(val, idx) + WHERE + _s3.val <> '' ) SELECT key, diff --git a/tests/test_sql_refsols/explode_21_snowflake.sql b/tests/test_sql_refsols/explode_21_snowflake.sql index 735b578bb..6fe6ffe64 100644 --- a/tests/test_sql_refsols/explode_21_snowflake.sql +++ b/tests/test_sql_refsols/explode_21_snowflake.sql @@ -16,9 +16,9 @@ WITH _q_0 AS ( FROM _q_0 AS _q_0 CROSS JOIN LATERAL SPLIT_TO_TABLE(_q_0.comment, '.') AS _s0 CROSS JOIN LATERAL SPLIT_TO_TABLE(_s0.value, ',') AS _s1 - CROSS JOIN LATERAL SPLIT_TO_TABLE(_s1.value, ' ') AS _s2 - JOIN LATERAL FLATTEN(REGEXP_EXTRACT_ALL(_s2.value, '.{1}')) AS _s3(seq, key, path, index, value, this) - ON _s3.value <> '' + CROSS JOIN LATERAL SPLIT_TO_TABLE(_s1.value, ' ') AS _s2, LATERAL FLATTEN(REGEXP_EXTRACT_ALL(_s2.value, '.{1}')) AS _s3(seq, key, path, index, value, this) + WHERE + _s3.value <> '' QUALIFY ROW_NUMBER() OVER (PARTITION BY _s1.index - 1, _s0.index - 1, key, _s2.index - 1 ORDER BY _s3.index - 1) = 1 ) diff --git a/tests/test_sql_refsols/explode_21_trino.sql b/tests/test_sql_refsols/explode_21_trino.sql index 3c04aecf6..352dae69f 100644 --- a/tests/test_sql_refsols/explode_21_trino.sql +++ b/tests/test_sql_refsols/explode_21_trino.sql @@ -18,9 +18,9 @@ WITH _q_0 AS ( FROM _q_0 AS _q_0 CROSS JOIN UNNEST(SPLIT(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) CROSS JOIN UNNEST(SPLIT(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx) - CROSS JOIN UNNEST(SPLIT(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) - JOIN UNNEST(SPLIT(_s2.val, '')) WITH ORDINALITY AS _s3(val, idx) - ON _s3.val <> '' + CROSS JOIN UNNEST(SPLIT(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx), UNNEST(SPLIT(_s2.val, '')) WITH ORDINALITY AS _s3(val, idx) + WHERE + _s3.val <> '' ) SELECT key, diff --git a/tests/testing_utilities.py b/tests/testing_utilities.py index 1ff7400aa..2c818ed4e 100644 --- a/tests/testing_utilities.py +++ b/tests/testing_utilities.py @@ -1659,7 +1659,7 @@ def run_e2e_test_to_table( "to_table did not return an UnqualifiedGeneratedCollection as expected" ) # Access the inner PyDoughUserGeneratedCollection to get columns - inner_collection = collection.user_collection + inner_collection = collection._parcel[0] assert isinstance(inner_collection, ViewGeneratedCollection), ( "to_table did not return a ViewGeneratedCollection as expected" ) From dafd2c12012fc5f003750d3e85696e09f7937ce6 Mon Sep 17 00:00:00 2001 From: knassre-bodo Date: Tue, 18 Aug 2026 10:06:44 -0700 Subject: [PATCH 31/44] Added documentation, fixed bugs with name collisions by qualifying unexploded data columns [RUN DIALECTS][RUN CI] --- documentation/dsl.md | 122 +++++++++++++++++- documentation/functions.md | 18 +++ pydough/sqlglot/sqlglot_relational_visitor.py | 1 + .../base_transform_bindings.py | 3 + .../databricks_transform_bindings.py | 8 +- .../duckdb_transform_bindings.py | 8 +- .../postgres_transform_bindings.py | 8 +- .../sf_transform_bindings.py | 28 +++- .../trino_transform_bindings.py | 8 +- .../explode_03_databricks.sql | 8 +- tests/test_sql_refsols/explode_03_duckdb.sql | 10 +- .../test_sql_refsols/explode_03_postgres.sql | 8 +- .../test_sql_refsols/explode_03_snowflake.sql | 8 +- tests/test_sql_refsols/explode_03_trino.sql | 8 +- .../explode_04_databricks.sql | 44 +++---- tests/test_sql_refsols/explode_04_duckdb.sql | 54 ++++---- .../test_sql_refsols/explode_04_postgres.sql | 44 +++---- .../test_sql_refsols/explode_04_snowflake.sql | 44 +++---- tests/test_sql_refsols/explode_04_trino.sql | 44 +++---- .../explode_07_databricks.sql | 14 +- tests/test_sql_refsols/explode_07_duckdb.sql | 16 +-- .../test_sql_refsols/explode_07_postgres.sql | 14 +- .../test_sql_refsols/explode_07_snowflake.sql | 14 +- tests/test_sql_refsols/explode_07_trino.sql | 14 +- .../explode_08_databricks.sql | 18 +-- tests/test_sql_refsols/explode_08_duckdb.sql | 26 ++-- .../test_sql_refsols/explode_08_postgres.sql | 18 +-- .../test_sql_refsols/explode_08_snowflake.sql | 18 +-- tests/test_sql_refsols/explode_08_trino.sql | 18 +-- .../explode_11_databricks.sql | 8 +- tests/test_sql_refsols/explode_11_duckdb.sql | 8 +- .../test_sql_refsols/explode_11_postgres.sql | 8 +- .../test_sql_refsols/explode_11_snowflake.sql | 8 +- tests/test_sql_refsols/explode_11_trino.sql | 8 +- .../explode_14_databricks.sql | 20 +-- tests/test_sql_refsols/explode_14_duckdb.sql | 28 ++-- .../test_sql_refsols/explode_14_postgres.sql | 20 +-- .../test_sql_refsols/explode_14_snowflake.sql | 20 +-- tests/test_sql_refsols/explode_14_trino.sql | 20 +-- .../explode_15_databricks.sql | 20 +-- tests/test_sql_refsols/explode_15_duckdb.sql | 28 ++-- .../test_sql_refsols/explode_15_postgres.sql | 20 +-- .../test_sql_refsols/explode_15_snowflake.sql | 20 +-- tests/test_sql_refsols/explode_15_trino.sql | 20 +-- .../explode_16_databricks.sql | 20 +-- tests/test_sql_refsols/explode_16_duckdb.sql | 28 ++-- .../test_sql_refsols/explode_16_postgres.sql | 20 +-- .../test_sql_refsols/explode_16_snowflake.sql | 20 +-- tests/test_sql_refsols/explode_16_trino.sql | 20 +-- .../explode_17_databricks.sql | 20 +-- tests/test_sql_refsols/explode_17_duckdb.sql | 28 ++-- .../test_sql_refsols/explode_17_postgres.sql | 20 +-- .../test_sql_refsols/explode_17_snowflake.sql | 20 +-- tests/test_sql_refsols/explode_17_trino.sql | 20 +-- .../explode_18_databricks.sql | 20 +-- tests/test_sql_refsols/explode_18_duckdb.sql | 28 ++-- .../test_sql_refsols/explode_18_postgres.sql | 20 +-- .../test_sql_refsols/explode_18_snowflake.sql | 20 +-- tests/test_sql_refsols/explode_18_trino.sql | 20 +-- .../explode_19_databricks.sql | 20 +-- tests/test_sql_refsols/explode_19_duckdb.sql | 28 ++-- .../test_sql_refsols/explode_19_postgres.sql | 20 +-- .../test_sql_refsols/explode_19_snowflake.sql | 20 +-- tests/test_sql_refsols/explode_19_trino.sql | 20 +-- .../explode_20_databricks.sql | 4 +- tests/test_sql_refsols/explode_20_duckdb.sql | 8 +- .../test_sql_refsols/explode_20_postgres.sql | 4 +- .../test_sql_refsols/explode_20_snowflake.sql | 4 +- tests/test_sql_refsols/explode_20_trino.sql | 4 +- .../explode_21_databricks.sql | 24 ++-- tests/test_sql_refsols/explode_21_duckdb.sql | 36 +++--- .../test_sql_refsols/explode_21_postgres.sql | 24 ++-- .../test_sql_refsols/explode_21_snowflake.sql | 29 +++-- tests/test_sql_refsols/explode_21_trino.sql | 24 ++-- 74 files changed, 841 insertions(+), 654 deletions(-) diff --git a/documentation/dsl.md b/documentation/dsl.md index f269ba404..4ccf95edf 100644 --- a/documentation/dsl.md +++ b/documentation/dsl.md @@ -19,6 +19,7 @@ This page describes the specification of the PyDough DSL. The specification incl * [SINGULAR](#singular) * [BEST](#best) * [CROSS](#cross) + * [EXPLODE](#explode) - [User Generated Collections](#user-generated-collections) * [range_collection](#range_collection) * [dataframe_collection](#dataframe_collection) @@ -36,7 +37,7 @@ This page describes the specification of the PyDough DSL. The specification incl ## Example Graph The examples in this document use a metadata graph (named `GRAPH`) with the following collections: -- `People`: records of every known person. Scalar properties: `first_name`, `middle_name`, `last_name`, `ssn`, `birth_date`, `email`, `current_address_id`. +- `People`: records of every known person. Scalar properties: `first_name`, `middle_name`, `last_name`, `ssn`, `birth_date`, `email`, `current_address_id`, `phone_numbers`. - `Addresses`: records of every known address. Scalar properties: `address_id`, `street_number`, `street_name`, `apartment`, `zip_code`, `city`, `state`. - `Packages`: records of every known package. Scalar properties: `package_id`, `customer_ssn`, `shipping_address_id`, `billing_address_id`, `order_date`, `arrival_date`, `package_cost`. @@ -1536,6 +1537,119 @@ People.CALCULATE(Packages=COUNT(People.packages)).CROSS(Packages) People.CROSS(Addresses).current_address ``` + +### EXPLODE + +A PyDough operation that explodes each row from a collection into multiple rows, i.e. from flattening a column of array data, or by splitting up a string column on a delimiter. The outputted collection will be a sub-collection of the original context containing the exploded data, and an optional indexed column keeping track of the indices of each value of the exploded data within a single row. This newly generated sub-collection has no other sub-collections from the original collection. The syntax for this operation is `collection.EXPLODE(...)`. `EXPLODE` has the following arguments: +- `data` (required): the expression from the current context being exploded (either an array or string). This expression cannot reference any sub-collections of the current context. +- `name` (required): the name of the collection created from the explosion operation (similar to `PARTITION`). +- `value_name` (required): a string literal declaring the name of the new column that will be used to store the exploded data. +- `index_name` (optional): a string literal declaring the name of the new column that will be used to store the indices of the exploded data. If not provided, this column is not generated. The `index_name` is required if `is_distinct` is False. The indices are 0-indexed. +- `version` (optional, default=`"array"`): either `"array"` or `"string"`, stating whether the data to explode is an array being flattened or a string being split on a delimiter. +- `delimiter` (optional): a string literal indicating the delimiter that should be used to split up the string if `version="string"`. If `delimiter` is an empty string, the string will be split into individual characters. +- `filtering` (optional, default=`True`): `True` if it is possible for not every row in the original collection to be preserved in the exploded sub-collection (i.e. if one of the arrays is empty), and `False` otherwise. +- `is_distinct` (optional, default=`False`): `True` if each row of the exploded data is unique within the set of all other values from that same original row of unexploded data, and `False` otherwise. An `index_name` can only be omitted if `is_distinct` is `True`. + +> [!IMPORTANT] +> This feature is only supported in certain dialects. It is currently supported for Snowflake, DataBricks, DuckDB, Postgres and Trino. + +**Good Example #1**: List out every phone number had by every person (assuming `phone_numbers` is an array of strings). + +```py +%%pydough +People.EXPLODE(phone_numbers, "numbers", value_name='phone_number', is_distinct=True) +``` + +**Good Example #2**: For each person, find the first phone number they have (assuming `phone_numbers` is an array of strings). + +```py +%%pydough +People.CALCULATE(first_name, last_name) + .EXPLODE(phone_numbers, "numbers", value_name='phone_number', index_name='idx', is_distinct=True) + .WHERE(idx == 0) + .CALCULATE(first_name, last_name, phone_number) +``` + +**Good Example #3**: For each person, count how many phone numbers they have (assuming `phone_numbers` is an array of strings). +```py +%%pydough +exploded_numbers = EXPLODE(phone_numbers, "numbers", value_name='phone_number', is_distinct=True) +People.CALCULATE(first_name, last_name, n_phone_numbers=COUNT(exploded_numbers)) +``` + +**Good Example #4**: List out every phone number had by every person (assuming `phone_numbers` is a string of comma separated values). + +```py +%%pydough +People.EXPLODE(phone_numbers, "numbers", value_name='phone_number', version="string", delimiter=",", is_distinct=True) +``` + +**Good Example #5**: List the number of times each character of the alphabet used within first names of people. + +```py +%%pydough +People.EXPLODE(LOWER(first_name), "characters", value_name='char', index_name="idx", version="string", delimiter="") + .PARTITION(name='letters', by=char) + .CALCULATE(char, n_uses=COUNT(characters)) +``` + +**Bad Example #1**: Missing the `name`. + +```py +%%pydough +People.EXPLODE(phone_numbers, value_name='phone_number', index_name="idx") +``` + +**Bad Example #2**: Missing the `value_name`. + +```py +%%pydough +People.EXPLODE(phone_numbers, "numbers", index_name="idx") +``` + +**Bad Example #3**: Missing the `index_name` when `is_distinct=True` is not provided. + +```py +%%pydough +People.EXPLODE(phone_numbers, "numbers", value_name='phone_number') +``` + +**Bad Example #4**: Providing an invalid `version` + +```py +%%pydough +People.EXPLODE(phone_numbers, "numbers", value_name='phone_number', index_name="idx", version="party") +``` + +**Bad Example #5**: Missing the `delimiter` when `version="string"` + +```py +%%pydough +People.EXPLODE(phone_numbers, "numbers", value_name='phone_number', index_name="idx", version="string") +``` + +**Bad Example #6**: Not providing a string literal for the delimiter. + +```py +%%pydough +People.EXPLODE(phone_numbers, "numbers", value_name='phone_number', index_name="idx", version="string", delimiter=first_name) +``` + +**Bad Example #7**: Attempting to access a sub-collection after exploding. + +```py +%%pydough +People.EXPLODE(phone_numbers, "numbers", value_name='phone_number', index_name="idx") + .packages +``` + +**Bad Example #8**: Accessing a sub-collection when declaring the data to explode. + +```py +%%pydough +People.EXPLODE(LISTOF(packages.package_cost), "costs", value_name='cost', index_name="idx") +``` + ## User Generated Collections @@ -1605,12 +1719,16 @@ The supported PyDough types for `dataframe_collection` are: - `NumericType`: includes float, integer, infinity, Nan. - `BooleanType`: True or False. - `StringType`: alphanumeric characters. -- `Datetype`: date and datetime. +- `DateType`: date and datetime. +- `ArrayType`: arrays of data. - `UnknownType`: used for all `None` columns. Note: MySQL by default does not support infinity values. When PyDough detects infinity value with `DatabaseDiatect.MYSQL` an error will be raised. +> [!IMPORTANT] +> `ArrayType` is only supported for certain dialects: Trino, Postgres, DuckDB, Databricks. + #### Example 1 ```python diff --git a/documentation/functions.md b/documentation/functions.md index 17be1d83c..f0d81d65e 100644 --- a/documentation/functions.md +++ b/documentation/functions.md @@ -72,6 +72,7 @@ Below is the list of every function/operator currently supported in PyDough as a * [HASNOT](#hasnot) * [VAR](#var) * [STD](#std) + * [LISTOF](#listof) - [Window Functions](#window-functions) * [RANKING](#ranking) * [PERCENTILE](#percentile) @@ -1089,6 +1090,23 @@ Parts.CALCULATE(std = STD(supply_records.supply_cost)) Parts.CALCULATE(std = STD(supply_records.supply_cost, type="sample")) ``` + + +### LISTOF + +The `LISTOF` function collects a set of values into an array. + +> [!IMPORTANT] +> This function is only supported in certain dialects. It is currently supported for Snowflake, DataBricks, DuckDB, Postgres and Trino. + +```py +# For each region, list the names of all nations inside that region +Regions.CALCULATE(region_name=name, nation_names=LISTOF(nations.name)) + +# For each customer, list the total quantities of all purchases made by that customer. +Customers.CALCULATE(customer_name=name, quantities=LISTOF(orders.lines.quantity)) +``` + ## Window Functions diff --git a/pydough/sqlglot/sqlglot_relational_visitor.py b/pydough/sqlglot/sqlglot_relational_visitor.py index fab7e4419..f34cfa53f 100644 --- a/pydough/sqlglot/sqlglot_relational_visitor.py +++ b/pydough/sqlglot/sqlglot_relational_visitor.py @@ -611,6 +611,7 @@ def visit_explode(self, explode: Explode) -> None: val_index, idx_index, self._generate_table_alias(), + self._generate_table_alias(), ) self._stack.append(query) diff --git a/pydough/sqlglot/transform_bindings/base_transform_bindings.py b/pydough/sqlglot/transform_bindings/base_transform_bindings.py index fc99b0e7c..8113a1ea8 100644 --- a/pydough/sqlglot/transform_bindings/base_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/base_transform_bindings.py @@ -2433,6 +2433,7 @@ def convert_explode( val_index: int | None, idx_index: int | None, lateral_alias: str, + subquery_alias: str, ) -> SQLGlotExpression: """ Converts a PyDough EXPLODE operation call to a SQLGlot expression that @@ -2451,6 +2452,8 @@ def convert_explode( the exploded data's index in the output. If None, not included. `lateral_alias`: A name given to a LATERAL operator, if needed, to avoid name collisions. + `subquery_alias`: A name given to the data before the LATERAL + operator, if needed, to avoid name collisions. Returns: A SQLGlotExpression representing the exploded data. diff --git a/pydough/sqlglot/transform_bindings/databricks_transform_bindings.py b/pydough/sqlglot/transform_bindings/databricks_transform_bindings.py index 0b5b99edc..cc3b527fa 100644 --- a/pydough/sqlglot/transform_bindings/databricks_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/databricks_transform_bindings.py @@ -110,6 +110,7 @@ def convert_explode( val_index: int | None, idx_index: int | None, lateral_alias: str, + subquery_alias: str, ) -> SQLGlotExpression: column_exprs: list[SQLGlotExpression] = [*exprs] if val_index is not None: @@ -149,7 +150,12 @@ def convert_explode( result = ( Select() .select(*column_exprs) - .from_(Subquery(this=input_expr)) + .from_( + Subquery( + this=input_expr, + alias=TableAlias(this=Identifier(this=subquery_alias)), + ) + ) .join( Lateral( this=explode_op, diff --git a/pydough/sqlglot/transform_bindings/duckdb_transform_bindings.py b/pydough/sqlglot/transform_bindings/duckdb_transform_bindings.py index 0033b5223..fc8889cc9 100644 --- a/pydough/sqlglot/transform_bindings/duckdb_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/duckdb_transform_bindings.py @@ -147,6 +147,7 @@ def convert_explode( val_index: int | None, idx_index: int | None, lateral_alias: str, + subquery_alias: str, ) -> SQLGlotExpression: column_exprs: list[SQLGlotExpression] = [*exprs] if val_index is not None: @@ -208,7 +209,12 @@ def convert_explode( result = ( Select() .select(*column_exprs) - .from_(Subquery(this=input_expr)) + .from_( + Subquery( + this=input_expr, + alias=TableAlias(this=Identifier(this=subquery_alias)), + ) + ) .join( Lateral( this=Subquery(this=Select().select(*explode_args)), diff --git a/pydough/sqlglot/transform_bindings/postgres_transform_bindings.py b/pydough/sqlglot/transform_bindings/postgres_transform_bindings.py index 0a94f4f87..09396937c 100644 --- a/pydough/sqlglot/transform_bindings/postgres_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/postgres_transform_bindings.py @@ -85,6 +85,7 @@ def convert_explode( val_index: int | None, idx_index: int | None, lateral_alias: str, + subquery_alias: str, ) -> SQLGlotExpression: column_exprs: list[SQLGlotExpression] = [*exprs] if val_index is not None: @@ -153,7 +154,12 @@ def convert_explode( result = ( Select() .select(*column_exprs) - .from_(Subquery(this=input_expr)) + .from_( + Subquery( + this=input_expr, + alias=TableAlias(this=Identifier(this=subquery_alias)), + ) + ) .join(Lateral(this=explode_op)) ) diff --git a/pydough/sqlglot/transform_bindings/sf_transform_bindings.py b/pydough/sqlglot/transform_bindings/sf_transform_bindings.py index 92d30f1d8..e2be6131c 100644 --- a/pydough/sqlglot/transform_bindings/sf_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/sf_transform_bindings.py @@ -256,8 +256,25 @@ def convert_explode( val_index: int | None, idx_index: int | None, lateral_alias: str, + subquery_alias: str, ) -> SQLGlotExpression: - column_exprs: list[SQLGlotExpression] = [*exprs] + # Rewrite all input expressions to point to the subquery + column_exprs: list[SQLGlotExpression] = [] + for expr in exprs: + if isinstance(expr, sqlglot_expressions.Identifier): + expr = sqlglot_expressions.Column( + this=sqlglot_expressions.Identifier(this=expr.this), + table=sqlglot_expressions.Identifier(this=subquery_alias), + ) + else: + for exp in expr.find_all(sqlglot_expressions.Identifier): + exp.replace( + sqlglot_expressions.Column( + this=sqlglot_expressions.Identifier(this=exp.this), + table=sqlglot_expressions.Identifier(this=subquery_alias), + ) + ) + column_exprs.append(expr) if val_index is not None: column_exprs[val_index] = sqlglot_expressions.Alias( this=sqlglot_expressions.Column( @@ -271,7 +288,7 @@ def convert_explode( this=sqlglot_expressions.Identifier(this="INDEX"), table=sqlglot_expressions.Identifier(this=lateral_alias), ) - if explode_spec.version == "string": + if explode_spec.version == "string" and explode_spec.delimiter != "": idx_expr = sqlglot_expressions.Sub( this=idx_expr, expression=sqlglot_expressions.Literal.number(1), @@ -312,7 +329,12 @@ def convert_explode( result = ( Select() .select(*column_exprs) - .from_(Subquery(this=input_expr)) + .from_( + Subquery( + this=input_expr, + alias=TableAlias(this=Identifier(this=subquery_alias)), + ) + ) .join( Lateral( this=explode_op, diff --git a/pydough/sqlglot/transform_bindings/trino_transform_bindings.py b/pydough/sqlglot/transform_bindings/trino_transform_bindings.py index cf21dfbaa..4cd586f18 100644 --- a/pydough/sqlglot/transform_bindings/trino_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/trino_transform_bindings.py @@ -133,6 +133,7 @@ def convert_explode( val_index: int | None, idx_index: int | None, lateral_alias: str, + subquery_alias: str, ) -> SQLGlotExpression: column_exprs: list[SQLGlotExpression] = [*exprs] if val_index is not None: @@ -186,7 +187,12 @@ def convert_explode( result = ( Select() .select(*column_exprs) - .from_(Subquery(this=input_expr)) + .from_( + Subquery( + this=input_expr, + alias=TableAlias(this=Identifier(this=subquery_alias)), + ) + ) .join(explode_op) ) diff --git a/tests/test_sql_refsols/explode_03_databricks.sql b/tests/test_sql_refsols/explode_03_databricks.sql index 4f533f7bd..dda3a8f05 100644 --- a/tests/test_sql_refsols/explode_03_databricks.sql +++ b/tests/test_sql_refsols/explode_03_databricks.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_name AS name FROM tpch.customer @@ -9,8 +9,8 @@ WITH _q_0 AS ( SELECT _s0.val, _s0.idx -FROM _q_0 AS _q_0 -CROSS JOIN LATERAL POSEXPLODE(SPLIT(_q_0.name, '\\Q#\\E')) AS _s0(idx, val) +FROM _s1 AS _s1 +CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s1.name, '\\Q#\\E')) AS _s0(idx, val) ORDER BY - _q_0.name, + _s1.name, 2 diff --git a/tests/test_sql_refsols/explode_03_duckdb.sql b/tests/test_sql_refsols/explode_03_duckdb.sql index cd0c9fc54..b855ef06e 100644 --- a/tests/test_sql_refsols/explode_03_duckdb.sql +++ b/tests/test_sql_refsols/explode_03_duckdb.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_name AS name FROM tpch.customer @@ -9,12 +9,12 @@ WITH _q_0 AS ( SELECT _s0.val, _s0.idx -FROM _q_0 AS _q_0 +FROM _s1 AS _s1 CROSS JOIN LATERAL ( SELECT - UNNEST(STR_SPLIT(_q_0.name, '#')) AS _col_0, - GENERATE_SUBSCRIPTS(STR_SPLIT(_q_0.name, '#'), 1) - 1 AS _col_1 + UNNEST(STR_SPLIT(_s1.name, '#')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_s1.name, '#'), 1) - 1 AS _col_1 ) AS _s0(val, idx) ORDER BY - _q_0.name NULLS FIRST, + _s1.name NULLS FIRST, 2 NULLS FIRST diff --git a/tests/test_sql_refsols/explode_03_postgres.sql b/tests/test_sql_refsols/explode_03_postgres.sql index f73b651fc..8b80108dc 100644 --- a/tests/test_sql_refsols/explode_03_postgres.sql +++ b/tests/test_sql_refsols/explode_03_postgres.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_name AS name FROM tpch.customer @@ -9,8 +9,8 @@ WITH _q_0 AS ( SELECT _s0.val, _s0.idx - 1 AS idx -FROM _q_0 AS _q_0 -CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_q_0.name, '#')) WITH ORDINALITY AS _s0(val, idx) +FROM _s1 AS _s1 +CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s1.name, '#')) WITH ORDINALITY AS _s0(val, idx) ORDER BY - _q_0.name NULLS FIRST, + _s1.name NULLS FIRST, 2 NULLS FIRST diff --git a/tests/test_sql_refsols/explode_03_snowflake.sql b/tests/test_sql_refsols/explode_03_snowflake.sql index 0e33b7a7a..fcfff6ddf 100644 --- a/tests/test_sql_refsols/explode_03_snowflake.sql +++ b/tests/test_sql_refsols/explode_03_snowflake.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_name AS name FROM tpch.customer @@ -9,8 +9,8 @@ WITH _q_0 AS ( SELECT _s0.value AS val, _s0.index - 1 AS idx -FROM _q_0 AS _q_0 -CROSS JOIN LATERAL SPLIT_TO_TABLE(_q_0.name, '#') AS _s0 +FROM _s1 AS _s1 +CROSS JOIN LATERAL SPLIT_TO_TABLE(_s1.name, '#') AS _s0 ORDER BY - _q_0.name NULLS FIRST, + _s1.name NULLS FIRST, 2 NULLS FIRST diff --git a/tests/test_sql_refsols/explode_03_trino.sql b/tests/test_sql_refsols/explode_03_trino.sql index e6674d08a..802f961a0 100644 --- a/tests/test_sql_refsols/explode_03_trino.sql +++ b/tests/test_sql_refsols/explode_03_trino.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_name AS name FROM tpch.customer @@ -9,8 +9,8 @@ WITH _q_0 AS ( SELECT _s0.val, _s0.idx - 1 AS idx -FROM _q_0 AS _q_0 -CROSS JOIN UNNEST(SPLIT(_q_0.name, '#')) WITH ORDINALITY AS _s0(val, idx) +FROM _s1 AS _s1 +CROSS JOIN UNNEST(SPLIT(_s1.name, '#')) WITH ORDINALITY AS _s0(val, idx) ORDER BY - _q_0.name NULLS FIRST, + _s1.name NULLS FIRST, 2 NULLS FIRST diff --git a/tests/test_sql_refsols/explode_04_databricks.sql b/tests/test_sql_refsols/explode_04_databricks.sql index 0d2069f02..618a19748 100644 --- a/tests/test_sql_refsols/explode_04_databricks.sql +++ b/tests/test_sql_refsols/explode_04_databricks.sql @@ -1,44 +1,44 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT r_regionkey AS key, r_name AS name FROM tpch.region -), _s2 AS ( +), _s3 AS ( SELECT - _q_0.key, + _s1.key, COUNT(*) AS n_rows - FROM _q_0 AS _q_0 - CROSS JOIN LATERAL POSEXPLODE(SPLIT(_q_0.name, '\\QE\\E')) AS _s0(idx, val) + FROM _s1 AS _s1 + CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s1.name, '\\QE\\E')) AS _s0(idx, val) GROUP BY 1 -), _s5 AS ( +), _s7 AS ( SELECT - _q_1.key, + _s5.key, COUNT(*) AS n_rows - FROM _q_0 AS _q_1 - CROSS JOIN LATERAL POSEXPLODE(SPLIT(_q_1.name, '\\QI\\E')) AS _s3(idx, val) + FROM _s1 AS _s5 + CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s5.name, '\\QI\\E')) AS _s4(idx, val) GROUP BY 1 -), _s8 AS ( +), _s11 AS ( SELECT - _q_2.key, + _s9.key, COUNT(*) AS n_rows - FROM _q_0 AS _q_2 - CROSS JOIN LATERAL POSEXPLODE(SPLIT(_q_2.name, '\\Q \\E')) AS _s6(idx, val) + FROM _s1 AS _s9 + CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s9.name, '\\Q \\E')) AS _s8(idx, val) GROUP BY 1 ) SELECT region.r_name AS region_name, - COALESCE(_s2.n_rows, 0) AS n_e_chunks, - COALESCE(_s5.n_rows, 0) AS n_i_chunks, - COALESCE(_s8.n_rows, 0) AS n_space_chunks + COALESCE(_s3.n_rows, 0) AS n_e_chunks, + COALESCE(_s7.n_rows, 0) AS n_i_chunks, + COALESCE(_s11.n_rows, 0) AS n_space_chunks FROM tpch.region AS region -LEFT JOIN _s2 AS _s2 - ON _s2.key = region.r_regionkey -LEFT JOIN _s5 AS _s5 - ON _s5.key = region.r_regionkey -LEFT JOIN _s8 AS _s8 - ON _s8.key = region.r_regionkey +LEFT JOIN _s3 AS _s3 + ON _s3.key = region.r_regionkey +LEFT JOIN _s7 AS _s7 + ON _s7.key = region.r_regionkey +LEFT JOIN _s11 AS _s11 + ON _s11.key = region.r_regionkey ORDER BY 1 diff --git a/tests/test_sql_refsols/explode_04_duckdb.sql b/tests/test_sql_refsols/explode_04_duckdb.sql index 392f55815..a77e54d0f 100644 --- a/tests/test_sql_refsols/explode_04_duckdb.sql +++ b/tests/test_sql_refsols/explode_04_duckdb.sql @@ -1,56 +1,56 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT r_regionkey AS key, r_name AS name FROM tpch.region -), _s2 AS ( +), _s3 AS ( SELECT - _q_0.key, + _s1.key, COUNT(*) AS n_rows - FROM _q_0 AS _q_0 + FROM _s1 AS _s1 CROSS JOIN LATERAL ( SELECT - UNNEST(STR_SPLIT(_q_0.name, 'E')) AS _col_0, - GENERATE_SUBSCRIPTS(STR_SPLIT(_q_0.name, 'E'), 1) - 1 AS _col_1 + UNNEST(STR_SPLIT(_s1.name, 'E')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_s1.name, 'E'), 1) - 1 AS _col_1 ) AS _s0(val, idx) GROUP BY 1 -), _s5 AS ( +), _s7 AS ( SELECT - _q_1.key, + _s5.key, COUNT(*) AS n_rows - FROM _q_0 AS _q_1 + FROM _s1 AS _s5 CROSS JOIN LATERAL ( SELECT - UNNEST(STR_SPLIT(_q_1.name, 'I')) AS _col_0, - GENERATE_SUBSCRIPTS(STR_SPLIT(_q_1.name, 'I'), 1) - 1 AS _col_1 - ) AS _s3(val, idx) + UNNEST(STR_SPLIT(_s5.name, 'I')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_s5.name, 'I'), 1) - 1 AS _col_1 + ) AS _s4(val, idx) GROUP BY 1 -), _s8 AS ( +), _s11 AS ( SELECT - _q_2.key, + _s9.key, COUNT(*) AS n_rows - FROM _q_0 AS _q_2 + FROM _s1 AS _s9 CROSS JOIN LATERAL ( SELECT - UNNEST(STR_SPLIT(_q_2.name, ' ')) AS _col_0, - GENERATE_SUBSCRIPTS(STR_SPLIT(_q_2.name, ' '), 1) - 1 AS _col_1 - ) AS _s6(val, idx) + UNNEST(STR_SPLIT(_s9.name, ' ')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_s9.name, ' '), 1) - 1 AS _col_1 + ) AS _s8(val, idx) GROUP BY 1 ) SELECT region.r_name AS region_name, - COALESCE(_s2.n_rows, 0) AS n_e_chunks, - COALESCE(_s5.n_rows, 0) AS n_i_chunks, - COALESCE(_s8.n_rows, 0) AS n_space_chunks + COALESCE(_s3.n_rows, 0) AS n_e_chunks, + COALESCE(_s7.n_rows, 0) AS n_i_chunks, + COALESCE(_s11.n_rows, 0) AS n_space_chunks FROM tpch.region AS region -LEFT JOIN _s2 AS _s2 - ON _s2.key = region.r_regionkey -LEFT JOIN _s5 AS _s5 - ON _s5.key = region.r_regionkey -LEFT JOIN _s8 AS _s8 - ON _s8.key = region.r_regionkey +LEFT JOIN _s3 AS _s3 + ON _s3.key = region.r_regionkey +LEFT JOIN _s7 AS _s7 + ON _s7.key = region.r_regionkey +LEFT JOIN _s11 AS _s11 + ON _s11.key = region.r_regionkey ORDER BY 1 NULLS FIRST diff --git a/tests/test_sql_refsols/explode_04_postgres.sql b/tests/test_sql_refsols/explode_04_postgres.sql index 76cf0a50c..999bd87ef 100644 --- a/tests/test_sql_refsols/explode_04_postgres.sql +++ b/tests/test_sql_refsols/explode_04_postgres.sql @@ -1,44 +1,44 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT r_regionkey AS key, r_name AS name FROM tpch.region -), _s2 AS ( +), _s3 AS ( SELECT - _q_0.key, + _s1.key, COUNT(*) AS n_rows - FROM _q_0 AS _q_0 - CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_q_0.name, 'E')) WITH ORDINALITY AS _s0(val, idx) + FROM _s1 AS _s1 + CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s1.name, 'E')) WITH ORDINALITY AS _s0(val, idx) GROUP BY 1 -), _s5 AS ( +), _s7 AS ( SELECT - _q_1.key, + _s5.key, COUNT(*) AS n_rows - FROM _q_0 AS _q_1 - CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_q_1.name, 'I')) WITH ORDINALITY AS _s3(val, idx) + FROM _s1 AS _s5 + CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s5.name, 'I')) WITH ORDINALITY AS _s4(val, idx) GROUP BY 1 -), _s8 AS ( +), _s11 AS ( SELECT - _q_2.key, + _s9.key, COUNT(*) AS n_rows - FROM _q_0 AS _q_2 - CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_q_2.name, ' ')) WITH ORDINALITY AS _s6(val, idx) + FROM _s1 AS _s9 + CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s9.name, ' ')) WITH ORDINALITY AS _s8(val, idx) GROUP BY 1 ) SELECT region.r_name AS region_name, - COALESCE(_s2.n_rows, 0) AS n_e_chunks, - COALESCE(_s5.n_rows, 0) AS n_i_chunks, - COALESCE(_s8.n_rows, 0) AS n_space_chunks + COALESCE(_s3.n_rows, 0) AS n_e_chunks, + COALESCE(_s7.n_rows, 0) AS n_i_chunks, + COALESCE(_s11.n_rows, 0) AS n_space_chunks FROM tpch.region AS region -LEFT JOIN _s2 AS _s2 - ON _s2.key = region.r_regionkey -LEFT JOIN _s5 AS _s5 - ON _s5.key = region.r_regionkey -LEFT JOIN _s8 AS _s8 - ON _s8.key = region.r_regionkey +LEFT JOIN _s3 AS _s3 + ON _s3.key = region.r_regionkey +LEFT JOIN _s7 AS _s7 + ON _s7.key = region.r_regionkey +LEFT JOIN _s11 AS _s11 + ON _s11.key = region.r_regionkey ORDER BY 1 NULLS FIRST diff --git a/tests/test_sql_refsols/explode_04_snowflake.sql b/tests/test_sql_refsols/explode_04_snowflake.sql index e357a1660..7b0b1955e 100644 --- a/tests/test_sql_refsols/explode_04_snowflake.sql +++ b/tests/test_sql_refsols/explode_04_snowflake.sql @@ -1,44 +1,44 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT r_regionkey AS key, r_name AS name FROM tpch.region -), _s2 AS ( +), _s3 AS ( SELECT - _q_0.key, + _s1.key, COUNT(*) AS n_rows - FROM _q_0 AS _q_0 - CROSS JOIN LATERAL SPLIT_TO_TABLE(_q_0.name, 'E') AS _s0 + FROM _s1 AS _s1 + CROSS JOIN LATERAL SPLIT_TO_TABLE(_s1.name, 'E') AS _s0 GROUP BY 1 -), _s5 AS ( +), _s7 AS ( SELECT - _q_1.key, + _s5.key, COUNT(*) AS n_rows - FROM _q_0 AS _q_1 - CROSS JOIN LATERAL SPLIT_TO_TABLE(_q_1.name, 'I') AS _s3 + FROM _s1 AS _s5 + CROSS JOIN LATERAL SPLIT_TO_TABLE(_s5.name, 'I') AS _s4 GROUP BY 1 -), _s8 AS ( +), _s11 AS ( SELECT - _q_2.key, + _s9.key, COUNT(*) AS n_rows - FROM _q_0 AS _q_2 - CROSS JOIN LATERAL SPLIT_TO_TABLE(_q_2.name, ' ') AS _s6 + FROM _s1 AS _s9 + CROSS JOIN LATERAL SPLIT_TO_TABLE(_s9.name, ' ') AS _s8 GROUP BY 1 ) SELECT region.r_name AS region_name, - COALESCE(_s2.n_rows, 0) AS n_e_chunks, - COALESCE(_s5.n_rows, 0) AS n_i_chunks, - COALESCE(_s8.n_rows, 0) AS n_space_chunks + COALESCE(_s3.n_rows, 0) AS n_e_chunks, + COALESCE(_s7.n_rows, 0) AS n_i_chunks, + COALESCE(_s11.n_rows, 0) AS n_space_chunks FROM tpch.region AS region -LEFT JOIN _s2 AS _s2 - ON _s2.key = region.r_regionkey -LEFT JOIN _s5 AS _s5 - ON _s5.key = region.r_regionkey -LEFT JOIN _s8 AS _s8 - ON _s8.key = region.r_regionkey +LEFT JOIN _s3 AS _s3 + ON _s3.key = region.r_regionkey +LEFT JOIN _s7 AS _s7 + ON _s7.key = region.r_regionkey +LEFT JOIN _s11 AS _s11 + ON _s11.key = region.r_regionkey ORDER BY 1 NULLS FIRST diff --git a/tests/test_sql_refsols/explode_04_trino.sql b/tests/test_sql_refsols/explode_04_trino.sql index a631c8fb3..cbef51700 100644 --- a/tests/test_sql_refsols/explode_04_trino.sql +++ b/tests/test_sql_refsols/explode_04_trino.sql @@ -1,44 +1,44 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT r_regionkey AS key, r_name AS name FROM tpch.region -), _s2 AS ( +), _s3 AS ( SELECT - _q_0.key, + _s1.key, COUNT(*) AS n_rows - FROM _q_0 AS _q_0 - CROSS JOIN UNNEST(SPLIT(_q_0.name, 'E')) WITH ORDINALITY AS _s0(val, idx) + FROM _s1 AS _s1 + CROSS JOIN UNNEST(SPLIT(_s1.name, 'E')) WITH ORDINALITY AS _s0(val, idx) GROUP BY 1 -), _s5 AS ( +), _s7 AS ( SELECT - _q_1.key, + _s5.key, COUNT(*) AS n_rows - FROM _q_0 AS _q_1 - CROSS JOIN UNNEST(SPLIT(_q_1.name, 'I')) WITH ORDINALITY AS _s3(val, idx) + FROM _s1 AS _s5 + CROSS JOIN UNNEST(SPLIT(_s5.name, 'I')) WITH ORDINALITY AS _s4(val, idx) GROUP BY 1 -), _s8 AS ( +), _s11 AS ( SELECT - _q_2.key, + _s9.key, COUNT(*) AS n_rows - FROM _q_0 AS _q_2 - CROSS JOIN UNNEST(SPLIT(_q_2.name, ' ')) WITH ORDINALITY AS _s6(val, idx) + FROM _s1 AS _s9 + CROSS JOIN UNNEST(SPLIT(_s9.name, ' ')) WITH ORDINALITY AS _s8(val, idx) GROUP BY 1 ) SELECT region.r_name AS region_name, - COALESCE(_s2.n_rows, 0) AS n_e_chunks, - COALESCE(_s5.n_rows, 0) AS n_i_chunks, - COALESCE(_s8.n_rows, 0) AS n_space_chunks + COALESCE(_s3.n_rows, 0) AS n_e_chunks, + COALESCE(_s7.n_rows, 0) AS n_i_chunks, + COALESCE(_s11.n_rows, 0) AS n_space_chunks FROM tpch.region AS region -LEFT JOIN _s2 AS _s2 - ON _s2.key = region.r_regionkey -LEFT JOIN _s5 AS _s5 - ON _s5.key = region.r_regionkey -LEFT JOIN _s8 AS _s8 - ON _s8.key = region.r_regionkey +LEFT JOIN _s3 AS _s3 + ON _s3.key = region.r_regionkey +LEFT JOIN _s7 AS _s7 + ON _s7.key = region.r_regionkey +LEFT JOIN _s11 AS _s11 + ON _s11.key = region.r_regionkey ORDER BY 1 NULLS FIRST diff --git a/tests/test_sql_refsols/explode_07_databricks.sql b/tests/test_sql_refsols/explode_07_databricks.sql index 23dc9ccd6..762d1c723 100644 --- a/tests/test_sql_refsols/explode_07_databricks.sql +++ b/tests/test_sql_refsols/explode_07_databricks.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_comment AS comment, c_custkey AS key @@ -8,13 +8,13 @@ WITH _q_0 AS ( LIMIT 3 ) SELECT - _q_0.key, + _s1.key, _s0.idx AS idx1, - _s1.idx AS idx2, - _s1.val AS val2 -FROM _q_0 AS _q_0 -CROSS JOIN LATERAL POSEXPLODE(SPLIT(_q_0.comment, '\\Q.\\E')) AS _s0(idx, val) -CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s0.val, '\\Q,\\E')) AS _s1(idx, val) + _s2.idx AS idx2, + _s2.val AS val2 +FROM _s1 AS _s1 +CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s1.comment, '\\Q.\\E')) AS _s0(idx, val) +CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s0.val, '\\Q,\\E')) AS _s2(idx, val) ORDER BY 1, 2, diff --git a/tests/test_sql_refsols/explode_07_duckdb.sql b/tests/test_sql_refsols/explode_07_duckdb.sql index 3457e18b1..08c7978f9 100644 --- a/tests/test_sql_refsols/explode_07_duckdb.sql +++ b/tests/test_sql_refsols/explode_07_duckdb.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_comment AS comment, c_custkey AS key @@ -8,21 +8,21 @@ WITH _q_0 AS ( LIMIT 3 ) SELECT - _q_0.key, + _s1.key, _s0.idx AS idx1, - _s1.idx AS idx2, - _s1.val AS val2 -FROM _q_0 AS _q_0 + _s2.idx AS idx2, + _s2.val AS val2 +FROM _s1 AS _s1 CROSS JOIN LATERAL ( SELECT - UNNEST(STR_SPLIT(_q_0.comment, '.')) AS _col_0, - GENERATE_SUBSCRIPTS(STR_SPLIT(_q_0.comment, '.'), 1) - 1 AS _col_1 + UNNEST(STR_SPLIT(_s1.comment, '.')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_s1.comment, '.'), 1) - 1 AS _col_1 ) AS _s0(val, idx) CROSS JOIN LATERAL ( SELECT UNNEST(STR_SPLIT(_s0.val, ',')) AS _col_0, GENERATE_SUBSCRIPTS(STR_SPLIT(_s0.val, ','), 1) - 1 AS _col_1 -) AS _s1(val, idx) +) AS _s2(val, idx) ORDER BY 1 NULLS FIRST, 2 NULLS FIRST, diff --git a/tests/test_sql_refsols/explode_07_postgres.sql b/tests/test_sql_refsols/explode_07_postgres.sql index 23e2ce3ba..d7db5cb8c 100644 --- a/tests/test_sql_refsols/explode_07_postgres.sql +++ b/tests/test_sql_refsols/explode_07_postgres.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_comment AS comment, c_custkey AS key @@ -8,13 +8,13 @@ WITH _q_0 AS ( LIMIT 3 ) SELECT - _q_0.key, + _s1.key, _s0.idx - 1 AS idx1, - _s1.idx - 1 AS idx2, - _s1.val AS val2 -FROM _q_0 AS _q_0 -CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) -CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx) + _s2.idx - 1 AS idx2, + _s2.val AS val2 +FROM _s1 AS _s1 +CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s1.comment, '.')) WITH ORDINALITY AS _s0(val, idx) +CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s0.val, ',')) WITH ORDINALITY AS _s2(val, idx) ORDER BY 1 NULLS FIRST, 2 NULLS FIRST, diff --git a/tests/test_sql_refsols/explode_07_snowflake.sql b/tests/test_sql_refsols/explode_07_snowflake.sql index 9d66b8cc6..7f5a84730 100644 --- a/tests/test_sql_refsols/explode_07_snowflake.sql +++ b/tests/test_sql_refsols/explode_07_snowflake.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_comment AS comment, c_custkey AS key @@ -8,13 +8,13 @@ WITH _q_0 AS ( LIMIT 3 ) SELECT - _q_0.key, + _s1.key, _s0.index - 1 AS idx1, - _s1.index - 1 AS idx2, - _s1.value AS val2 -FROM _q_0 AS _q_0 -CROSS JOIN LATERAL SPLIT_TO_TABLE(_q_0.comment, '.') AS _s0 -CROSS JOIN LATERAL SPLIT_TO_TABLE(_s0.value, ',') AS _s1 + _s2.index - 1 AS idx2, + _s2.value AS val2 +FROM _s1 AS _s1 +CROSS JOIN LATERAL SPLIT_TO_TABLE(_s1.comment, '.') AS _s0 +CROSS JOIN LATERAL SPLIT_TO_TABLE(_s0.value, ',') AS _s2 ORDER BY 1 NULLS FIRST, 2 NULLS FIRST, diff --git a/tests/test_sql_refsols/explode_07_trino.sql b/tests/test_sql_refsols/explode_07_trino.sql index 888e37a3b..1686b936c 100644 --- a/tests/test_sql_refsols/explode_07_trino.sql +++ b/tests/test_sql_refsols/explode_07_trino.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_comment AS comment, c_custkey AS key @@ -8,13 +8,13 @@ WITH _q_0 AS ( LIMIT 3 ) SELECT - _q_0.key, + _s1.key, _s0.idx - 1 AS idx1, - _s1.idx - 1 AS idx2, - _s1.val AS val2 -FROM _q_0 AS _q_0 -CROSS JOIN UNNEST(SPLIT(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) -CROSS JOIN UNNEST(SPLIT(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx) + _s2.idx - 1 AS idx2, + _s2.val AS val2 +FROM _s1 AS _s1 +CROSS JOIN UNNEST(SPLIT(_s1.comment, '.')) WITH ORDINALITY AS _s0(val, idx) +CROSS JOIN UNNEST(SPLIT(_s0.val, ',')) WITH ORDINALITY AS _s2(val, idx) ORDER BY 1 NULLS FIRST, 2 NULLS FIRST, diff --git a/tests/test_sql_refsols/explode_08_databricks.sql b/tests/test_sql_refsols/explode_08_databricks.sql index a0dc2a2d8..092926ce7 100644 --- a/tests/test_sql_refsols/explode_08_databricks.sql +++ b/tests/test_sql_refsols/explode_08_databricks.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_comment AS comment, c_custkey AS key @@ -8,16 +8,16 @@ WITH _q_0 AS ( LIMIT 3 ) SELECT - _q_0.key, + _s1.key, _s0.idx AS idx1, - _s1.idx AS idx2, - _s2.idx AS idx3, - _s2.val AS val3 -FROM _q_0 AS _q_0 -CROSS JOIN LATERAL POSEXPLODE(SPLIT(_q_0.comment, '\\Q.\\E')) AS _s0(idx, val) -CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s0.val, '\\Q \\E')) AS _s1(idx, val), LATERAL POSEXPLODE(SPLIT(_s1.val, '\\Q,\\E')) AS _s2(idx, val) + _s2.idx AS idx2, + _s4.idx AS idx3, + _s4.val AS val3 +FROM _s1 AS _s1 +CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s1.comment, '\\Q.\\E')) AS _s0(idx, val) +CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s0.val, '\\Q \\E')) AS _s2(idx, val), LATERAL POSEXPLODE(SPLIT(_s2.val, '\\Q,\\E')) AS _s4(idx, val) WHERE - _s2.val <> '' + _s4.val <> '' ORDER BY 1, 2, diff --git a/tests/test_sql_refsols/explode_08_duckdb.sql b/tests/test_sql_refsols/explode_08_duckdb.sql index f8f310282..30a8e3fca 100644 --- a/tests/test_sql_refsols/explode_08_duckdb.sql +++ b/tests/test_sql_refsols/explode_08_duckdb.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_comment AS comment, c_custkey AS key @@ -8,28 +8,28 @@ WITH _q_0 AS ( LIMIT 3 ) SELECT - _q_0.key, + _s1.key, _s0.idx AS idx1, - _s1.idx AS idx2, - _s2.idx AS idx3, - _s2.val AS val3 -FROM _q_0 AS _q_0 + _s2.idx AS idx2, + _s4.idx AS idx3, + _s4.val AS val3 +FROM _s1 AS _s1 CROSS JOIN LATERAL ( SELECT - UNNEST(STR_SPLIT(_q_0.comment, '.')) AS _col_0, - GENERATE_SUBSCRIPTS(STR_SPLIT(_q_0.comment, '.'), 1) - 1 AS _col_1 + UNNEST(STR_SPLIT(_s1.comment, '.')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_s1.comment, '.'), 1) - 1 AS _col_1 ) AS _s0(val, idx) CROSS JOIN LATERAL ( SELECT UNNEST(STR_SPLIT(_s0.val, ' ')) AS _col_0, GENERATE_SUBSCRIPTS(STR_SPLIT(_s0.val, ' '), 1) - 1 AS _col_1 -) AS _s1(val, idx), LATERAL ( +) AS _s2(val, idx), LATERAL ( SELECT - UNNEST(STR_SPLIT(_s1.val, ',')) AS _col_0, - GENERATE_SUBSCRIPTS(STR_SPLIT(_s1.val, ','), 1) - 1 AS _col_1 -) AS _s2(val, idx) + UNNEST(STR_SPLIT(_s2.val, ',')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_s2.val, ','), 1) - 1 AS _col_1 +) AS _s4(val, idx) WHERE - _s2.val <> '' + _s4.val <> '' ORDER BY 1 NULLS FIRST, 2 NULLS FIRST, diff --git a/tests/test_sql_refsols/explode_08_postgres.sql b/tests/test_sql_refsols/explode_08_postgres.sql index 702087490..abbbffa78 100644 --- a/tests/test_sql_refsols/explode_08_postgres.sql +++ b/tests/test_sql_refsols/explode_08_postgres.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_comment AS comment, c_custkey AS key @@ -8,16 +8,16 @@ WITH _q_0 AS ( LIMIT 3 ) SELECT - _q_0.key, + _s1.key, _s0.idx - 1 AS idx1, - _s1.idx - 1 AS idx2, - _s2.idx - 1 AS idx3, - _s2.val AS val3 -FROM _q_0 AS _q_0 -CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) -CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s0.val, ' ')) WITH ORDINALITY AS _s1(val, idx), LATERAL UNNEST(STRING_TO_ARRAY(_s1.val, ',')) WITH ORDINALITY AS _s2(val, idx) + _s2.idx - 1 AS idx2, + _s4.idx - 1 AS idx3, + _s4.val AS val3 +FROM _s1 AS _s1 +CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s1.comment, '.')) WITH ORDINALITY AS _s0(val, idx) +CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s0.val, ' ')) WITH ORDINALITY AS _s2(val, idx), LATERAL UNNEST(STRING_TO_ARRAY(_s2.val, ',')) WITH ORDINALITY AS _s4(val, idx) WHERE - _s2.val <> '' + _s4.val <> '' ORDER BY 1 NULLS FIRST, 2 NULLS FIRST, diff --git a/tests/test_sql_refsols/explode_08_snowflake.sql b/tests/test_sql_refsols/explode_08_snowflake.sql index d3dcfb595..6f06a63b2 100644 --- a/tests/test_sql_refsols/explode_08_snowflake.sql +++ b/tests/test_sql_refsols/explode_08_snowflake.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_comment AS comment, c_custkey AS key @@ -8,16 +8,16 @@ WITH _q_0 AS ( LIMIT 3 ) SELECT - _q_0.key, + _s1.key, _s0.index - 1 AS idx1, - _s1.index - 1 AS idx2, - _s2.index - 1 AS idx3, - _s2.value AS val3 -FROM _q_0 AS _q_0 -CROSS JOIN LATERAL SPLIT_TO_TABLE(_q_0.comment, '.') AS _s0 -CROSS JOIN LATERAL SPLIT_TO_TABLE(_s0.value, ' ') AS _s1, LATERAL SPLIT_TO_TABLE(_s1.value, ',') AS _s2 + _s2.index - 1 AS idx2, + _s4.index - 1 AS idx3, + _s4.value AS val3 +FROM _s1 AS _s1 +CROSS JOIN LATERAL SPLIT_TO_TABLE(_s1.comment, '.') AS _s0 +CROSS JOIN LATERAL SPLIT_TO_TABLE(_s0.value, ' ') AS _s2, LATERAL SPLIT_TO_TABLE(_s2.value, ',') AS _s4 WHERE - _s2.value <> '' + _s4.value <> '' ORDER BY 1 NULLS FIRST, 2 NULLS FIRST, diff --git a/tests/test_sql_refsols/explode_08_trino.sql b/tests/test_sql_refsols/explode_08_trino.sql index a66d99328..9fe0f9af0 100644 --- a/tests/test_sql_refsols/explode_08_trino.sql +++ b/tests/test_sql_refsols/explode_08_trino.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_comment AS comment, c_custkey AS key @@ -8,16 +8,16 @@ WITH _q_0 AS ( LIMIT 3 ) SELECT - _q_0.key, + _s1.key, _s0.idx - 1 AS idx1, - _s1.idx - 1 AS idx2, - _s2.idx - 1 AS idx3, - _s2.val AS val3 -FROM _q_0 AS _q_0 -CROSS JOIN UNNEST(SPLIT(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) -CROSS JOIN UNNEST(SPLIT(_s0.val, ' ')) WITH ORDINALITY AS _s1(val, idx), UNNEST(SPLIT(_s1.val, ',')) WITH ORDINALITY AS _s2(val, idx) + _s2.idx - 1 AS idx2, + _s4.idx - 1 AS idx3, + _s4.val AS val3 +FROM _s1 AS _s1 +CROSS JOIN UNNEST(SPLIT(_s1.comment, '.')) WITH ORDINALITY AS _s0(val, idx) +CROSS JOIN UNNEST(SPLIT(_s0.val, ' ')) WITH ORDINALITY AS _s2(val, idx), UNNEST(SPLIT(_s2.val, ',')) WITH ORDINALITY AS _s4(val, idx) WHERE - _s2.val <> '' + _s4.val <> '' ORDER BY 1 NULLS FIRST, 2 NULLS FIRST, diff --git a/tests/test_sql_refsols/explode_11_databricks.sql b/tests/test_sql_refsols/explode_11_databricks.sql index 7293b56b5..f208127c4 100644 --- a/tests/test_sql_refsols/explode_11_databricks.sql +++ b/tests/test_sql_refsols/explode_11_databricks.sql @@ -1,17 +1,17 @@ -WITH _s3 AS ( +WITH _s5 AS ( SELECT DISTINCT - _s1.val AS cust_word + _s2.val AS cust_word FROM tpch.customer AS customer CROSS JOIN LATERAL POSEXPLODE( SPLIT( TRIM(' ' FROM REPLACE(REPLACE(REPLACE(REPLACE(customer.c_comment, ';', ''), ',', ''), ':', ''), '.', '')), '\\Q \\E' ) - ) AS _s1(idx, val) + ) AS _s2(idx, val) ), _u_0 AS ( SELECT cust_word AS _u_1 - FROM _s3 + FROM _s5 GROUP BY 1 ) diff --git a/tests/test_sql_refsols/explode_11_duckdb.sql b/tests/test_sql_refsols/explode_11_duckdb.sql index 75d8e521b..cda781613 100644 --- a/tests/test_sql_refsols/explode_11_duckdb.sql +++ b/tests/test_sql_refsols/explode_11_duckdb.sql @@ -1,6 +1,6 @@ -WITH _s3 AS ( +WITH _s5 AS ( SELECT DISTINCT - _s1.val AS cust_word + _s2.val AS cust_word FROM tpch.customer AS customer CROSS JOIN LATERAL ( SELECT @@ -23,11 +23,11 @@ WITH _s3 AS ( ), 1 ) - 1 AS _col_1 - ) AS _s1(val, idx) + ) AS _s2(val, idx) ), _u_0 AS ( SELECT cust_word AS _u_1 - FROM _s3 + FROM _s5 GROUP BY 1 ) diff --git a/tests/test_sql_refsols/explode_11_postgres.sql b/tests/test_sql_refsols/explode_11_postgres.sql index 25b4a35ff..77026e278 100644 --- a/tests/test_sql_refsols/explode_11_postgres.sql +++ b/tests/test_sql_refsols/explode_11_postgres.sql @@ -1,15 +1,15 @@ -WITH _s3 AS ( +WITH _s5 AS ( SELECT DISTINCT - _s1.val AS cust_word + _s2.val AS cust_word FROM tpch.customer AS customer CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY( TRIM(' ' FROM REPLACE(REPLACE(REPLACE(REPLACE(customer.c_comment, ';', ''), ',', ''), ':', ''), '.', '')), ' ' - )) WITH ORDINALITY AS _s1(val, idx) + )) WITH ORDINALITY AS _s2(val, idx) ), _u_0 AS ( SELECT cust_word AS _u_1 - FROM _s3 + FROM _s5 GROUP BY 1 ) diff --git a/tests/test_sql_refsols/explode_11_snowflake.sql b/tests/test_sql_refsols/explode_11_snowflake.sql index 020bd0ace..01ea5db99 100644 --- a/tests/test_sql_refsols/explode_11_snowflake.sql +++ b/tests/test_sql_refsols/explode_11_snowflake.sql @@ -1,6 +1,6 @@ -WITH _s3 AS ( +WITH _s5 AS ( SELECT DISTINCT - _s1.value AS cust_word + _s2.value AS cust_word FROM tpch.customer AS customer CROSS JOIN LATERAL SPLIT_TO_TABLE( TRIM( @@ -8,11 +8,11 @@ WITH _s3 AS ( ' ' ), ' ' - ) AS _s1 + ) AS _s2 ), _u_0 AS ( SELECT cust_word AS _u_1 - FROM _s3 + FROM _s5 GROUP BY 1 ) diff --git a/tests/test_sql_refsols/explode_11_trino.sql b/tests/test_sql_refsols/explode_11_trino.sql index f9ef9f905..483a0d81e 100644 --- a/tests/test_sql_refsols/explode_11_trino.sql +++ b/tests/test_sql_refsols/explode_11_trino.sql @@ -1,15 +1,15 @@ -WITH _s3 AS ( +WITH _s5 AS ( SELECT DISTINCT - _s1.val AS cust_word + _s2.val AS cust_word FROM tpch.customer AS customer CROSS JOIN UNNEST(SPLIT( TRIM(' ' FROM REPLACE(REPLACE(REPLACE(REPLACE(customer.c_comment, ';', ''), ',', ''), ':', ''), '.', '')), ' ' - )) WITH ORDINALITY AS _s1(val, idx) + )) WITH ORDINALITY AS _s2(val, idx) ), _u_0 AS ( SELECT cust_word AS _u_1 - FROM _s3 + FROM _s5 GROUP BY 1 ) diff --git a/tests/test_sql_refsols/explode_14_databricks.sql b/tests/test_sql_refsols/explode_14_databricks.sql index baafdd793..6c9467b13 100644 --- a/tests/test_sql_refsols/explode_14_databricks.sql +++ b/tests/test_sql_refsols/explode_14_databricks.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_comment AS comment, c_custkey AS key @@ -9,17 +9,17 @@ WITH _q_0 AS ( ), _t0 AS ( SELECT _s0.idx AS idx1, - _s1.idx AS idx2, - _s2.idx AS idx3, - _q_0.key, - _s2.val AS val3 - FROM _q_0 AS _q_0 - CROSS JOIN LATERAL POSEXPLODE(SPLIT(_q_0.comment, '\\Q.\\E')) AS _s0(idx, val) - CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s0.val, '\\Q,\\E')) AS _s1(idx, val), LATERAL POSEXPLODE(SPLIT(_s1.val, '\\Q \\E')) AS _s2(idx, val) + _s2.idx AS idx2, + _s4.idx AS idx3, + _s1.key, + _s4.val AS val3 + FROM _s1 AS _s1 + CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s1.comment, '\\Q.\\E')) AS _s0(idx, val) + CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s0.val, '\\Q,\\E')) AS _s2(idx, val), LATERAL POSEXPLODE(SPLIT(_s2.val, '\\Q \\E')) AS _s4(idx, val) WHERE - _s2.val <> '' + _s4.val <> '' QUALIFY - ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.idx NULLS LAST, _s1.idx NULLS LAST, _s2.idx NULLS LAST) = 1 + ROW_NUMBER() OVER (PARTITION BY _s1.key ORDER BY _s0.idx NULLS LAST, _s2.idx NULLS LAST, _s4.idx NULLS LAST) = 1 ) SELECT key, diff --git a/tests/test_sql_refsols/explode_14_duckdb.sql b/tests/test_sql_refsols/explode_14_duckdb.sql index fc5f04f62..330592ec8 100644 --- a/tests/test_sql_refsols/explode_14_duckdb.sql +++ b/tests/test_sql_refsols/explode_14_duckdb.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_comment AS comment, c_custkey AS key @@ -9,29 +9,29 @@ WITH _q_0 AS ( ), _t0 AS ( SELECT _s0.idx AS idx1, - _s1.idx AS idx2, - _s2.idx AS idx3, - _q_0.key, - _s2.val AS val3 - FROM _q_0 AS _q_0 + _s2.idx AS idx2, + _s4.idx AS idx3, + _s1.key, + _s4.val AS val3 + FROM _s1 AS _s1 CROSS JOIN LATERAL ( SELECT - UNNEST(STR_SPLIT(_q_0.comment, '.')) AS _col_0, - GENERATE_SUBSCRIPTS(STR_SPLIT(_q_0.comment, '.'), 1) - 1 AS _col_1 + UNNEST(STR_SPLIT(_s1.comment, '.')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_s1.comment, '.'), 1) - 1 AS _col_1 ) AS _s0(val, idx) CROSS JOIN LATERAL ( SELECT UNNEST(STR_SPLIT(_s0.val, ',')) AS _col_0, GENERATE_SUBSCRIPTS(STR_SPLIT(_s0.val, ','), 1) - 1 AS _col_1 - ) AS _s1(val, idx), LATERAL ( + ) AS _s2(val, idx), LATERAL ( SELECT - UNNEST(STR_SPLIT(_s1.val, ' ')) AS _col_0, - GENERATE_SUBSCRIPTS(STR_SPLIT(_s1.val, ' '), 1) - 1 AS _col_1 - ) AS _s2(val, idx) + UNNEST(STR_SPLIT(_s2.val, ' ')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_s2.val, ' '), 1) - 1 AS _col_1 + ) AS _s4(val, idx) WHERE - _s2.val <> '' + _s4.val <> '' QUALIFY - ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.idx, _s1.idx, _s2.idx) = 1 + ROW_NUMBER() OVER (PARTITION BY _s1.key ORDER BY _s0.idx, _s2.idx, _s4.idx) = 1 ) SELECT key, diff --git a/tests/test_sql_refsols/explode_14_postgres.sql b/tests/test_sql_refsols/explode_14_postgres.sql index 6daa460db..e1c641bb3 100644 --- a/tests/test_sql_refsols/explode_14_postgres.sql +++ b/tests/test_sql_refsols/explode_14_postgres.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_comment AS comment, c_custkey AS key @@ -9,16 +9,16 @@ WITH _q_0 AS ( ), _t AS ( SELECT _s0.idx - 1 AS idx1, - _s1.idx - 1 AS idx2, - _s2.idx - 1 AS idx3, - _q_0.key, - _s2.val AS val3, - ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.idx - 1, _s1.idx - 1, _s2.idx - 1) AS _w - FROM _q_0 AS _q_0 - CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) - CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx), LATERAL UNNEST(STRING_TO_ARRAY(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) + _s2.idx - 1 AS idx2, + _s4.idx - 1 AS idx3, + _s1.key, + _s4.val AS val3, + ROW_NUMBER() OVER (PARTITION BY _s1.key ORDER BY _s0.idx - 1, _s2.idx - 1, _s4.idx - 1) AS _w + FROM _s1 AS _s1 + CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s1.comment, '.')) WITH ORDINALITY AS _s0(val, idx) + CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s0.val, ',')) WITH ORDINALITY AS _s2(val, idx), LATERAL UNNEST(STRING_TO_ARRAY(_s2.val, ' ')) WITH ORDINALITY AS _s4(val, idx) WHERE - _s2.val <> '' + _s4.val <> '' ) SELECT key, diff --git a/tests/test_sql_refsols/explode_14_snowflake.sql b/tests/test_sql_refsols/explode_14_snowflake.sql index 325a8a493..ef639a3c8 100644 --- a/tests/test_sql_refsols/explode_14_snowflake.sql +++ b/tests/test_sql_refsols/explode_14_snowflake.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_comment AS comment, c_custkey AS key @@ -9,17 +9,17 @@ WITH _q_0 AS ( ), _t0 AS ( SELECT _s0.index - 1 AS idx1, - _s1.index - 1 AS idx2, - _s2.index - 1 AS idx3, - _q_0.key, - _s2.value AS val3 - FROM _q_0 AS _q_0 - CROSS JOIN LATERAL SPLIT_TO_TABLE(_q_0.comment, '.') AS _s0 - CROSS JOIN LATERAL SPLIT_TO_TABLE(_s0.value, ',') AS _s1, LATERAL SPLIT_TO_TABLE(_s1.value, ' ') AS _s2 + _s2.index - 1 AS idx2, + _s4.index - 1 AS idx3, + _s1.key, + _s4.value AS val3 + FROM _s1 AS _s1 + CROSS JOIN LATERAL SPLIT_TO_TABLE(_s1.comment, '.') AS _s0 + CROSS JOIN LATERAL SPLIT_TO_TABLE(_s0.value, ',') AS _s2, LATERAL SPLIT_TO_TABLE(_s2.value, ' ') AS _s4 WHERE - _s2.value <> '' + _s4.value <> '' QUALIFY - ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.index - 1, _s1.index - 1, _s2.index - 1) = 1 + ROW_NUMBER() OVER (PARTITION BY _s1.key ORDER BY _s0.index - 1, _s2.index - 1, _s4.index - 1) = 1 ) SELECT key, diff --git a/tests/test_sql_refsols/explode_14_trino.sql b/tests/test_sql_refsols/explode_14_trino.sql index 03d55684c..aa380d3e0 100644 --- a/tests/test_sql_refsols/explode_14_trino.sql +++ b/tests/test_sql_refsols/explode_14_trino.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_comment AS comment, c_custkey AS key @@ -9,16 +9,16 @@ WITH _q_0 AS ( ), _t AS ( SELECT _s0.idx - 1 AS idx1, - _s1.idx - 1 AS idx2, - _s2.idx - 1 AS idx3, - _q_0.key, - _s2.val AS val3, - ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.idx - 1, _s1.idx - 1, _s2.idx - 1) AS _w - FROM _q_0 AS _q_0 - CROSS JOIN UNNEST(SPLIT(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) - CROSS JOIN UNNEST(SPLIT(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx), UNNEST(SPLIT(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) + _s2.idx - 1 AS idx2, + _s4.idx - 1 AS idx3, + _s1.key, + _s4.val AS val3, + ROW_NUMBER() OVER (PARTITION BY _s1.key ORDER BY _s0.idx - 1, _s2.idx - 1, _s4.idx - 1) AS _w + FROM _s1 AS _s1 + CROSS JOIN UNNEST(SPLIT(_s1.comment, '.')) WITH ORDINALITY AS _s0(val, idx) + CROSS JOIN UNNEST(SPLIT(_s0.val, ',')) WITH ORDINALITY AS _s2(val, idx), UNNEST(SPLIT(_s2.val, ' ')) WITH ORDINALITY AS _s4(val, idx) WHERE - _s2.val <> '' + _s4.val <> '' ) SELECT key, diff --git a/tests/test_sql_refsols/explode_15_databricks.sql b/tests/test_sql_refsols/explode_15_databricks.sql index b84591570..10617c86f 100644 --- a/tests/test_sql_refsols/explode_15_databricks.sql +++ b/tests/test_sql_refsols/explode_15_databricks.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_comment AS comment, c_custkey AS key @@ -9,17 +9,17 @@ WITH _q_0 AS ( ), _t0 AS ( SELECT _s0.idx AS idx1, - _s1.idx AS idx2, - _s2.idx AS idx3, - _q_0.key, - _s2.val AS val3 - FROM _q_0 AS _q_0 - CROSS JOIN LATERAL POSEXPLODE(SPLIT(_q_0.comment, '\\Q.\\E')) AS _s0(idx, val) - CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s0.val, '\\Q,\\E')) AS _s1(idx, val), LATERAL POSEXPLODE(SPLIT(_s1.val, '\\Q \\E')) AS _s2(idx, val) + _s2.idx AS idx2, + _s4.idx AS idx3, + _s1.key, + _s4.val AS val3 + FROM _s1 AS _s1 + CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s1.comment, '\\Q.\\E')) AS _s0(idx, val) + CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s0.val, '\\Q,\\E')) AS _s2(idx, val), LATERAL POSEXPLODE(SPLIT(_s2.val, '\\Q \\E')) AS _s4(idx, val) WHERE - _s2.val <> '' + _s4.val <> '' QUALIFY - ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.idx DESC NULLS FIRST, _s1.idx NULLS LAST, _s2.idx NULLS LAST) = 1 + ROW_NUMBER() OVER (PARTITION BY _s1.key ORDER BY _s0.idx DESC NULLS FIRST, _s2.idx NULLS LAST, _s4.idx NULLS LAST) = 1 ) SELECT key, diff --git a/tests/test_sql_refsols/explode_15_duckdb.sql b/tests/test_sql_refsols/explode_15_duckdb.sql index 73ad5c72b..a1c4ba384 100644 --- a/tests/test_sql_refsols/explode_15_duckdb.sql +++ b/tests/test_sql_refsols/explode_15_duckdb.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_comment AS comment, c_custkey AS key @@ -9,29 +9,29 @@ WITH _q_0 AS ( ), _t0 AS ( SELECT _s0.idx AS idx1, - _s1.idx AS idx2, - _s2.idx AS idx3, - _q_0.key, - _s2.val AS val3 - FROM _q_0 AS _q_0 + _s2.idx AS idx2, + _s4.idx AS idx3, + _s1.key, + _s4.val AS val3 + FROM _s1 AS _s1 CROSS JOIN LATERAL ( SELECT - UNNEST(STR_SPLIT(_q_0.comment, '.')) AS _col_0, - GENERATE_SUBSCRIPTS(STR_SPLIT(_q_0.comment, '.'), 1) - 1 AS _col_1 + UNNEST(STR_SPLIT(_s1.comment, '.')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_s1.comment, '.'), 1) - 1 AS _col_1 ) AS _s0(val, idx) CROSS JOIN LATERAL ( SELECT UNNEST(STR_SPLIT(_s0.val, ',')) AS _col_0, GENERATE_SUBSCRIPTS(STR_SPLIT(_s0.val, ','), 1) - 1 AS _col_1 - ) AS _s1(val, idx), LATERAL ( + ) AS _s2(val, idx), LATERAL ( SELECT - UNNEST(STR_SPLIT(_s1.val, ' ')) AS _col_0, - GENERATE_SUBSCRIPTS(STR_SPLIT(_s1.val, ' '), 1) - 1 AS _col_1 - ) AS _s2(val, idx) + UNNEST(STR_SPLIT(_s2.val, ' ')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_s2.val, ' '), 1) - 1 AS _col_1 + ) AS _s4(val, idx) WHERE - _s2.val <> '' + _s4.val <> '' QUALIFY - ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.idx DESC NULLS FIRST, _s1.idx, _s2.idx) = 1 + ROW_NUMBER() OVER (PARTITION BY _s1.key ORDER BY _s0.idx DESC NULLS FIRST, _s2.idx, _s4.idx) = 1 ) SELECT key, diff --git a/tests/test_sql_refsols/explode_15_postgres.sql b/tests/test_sql_refsols/explode_15_postgres.sql index 663ca15d7..70c86c6c8 100644 --- a/tests/test_sql_refsols/explode_15_postgres.sql +++ b/tests/test_sql_refsols/explode_15_postgres.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_comment AS comment, c_custkey AS key @@ -9,16 +9,16 @@ WITH _q_0 AS ( ), _t AS ( SELECT _s0.idx - 1 AS idx1, - _s1.idx - 1 AS idx2, - _s2.idx - 1 AS idx3, - _q_0.key, - _s2.val AS val3, - ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.idx - 1 DESC, _s1.idx - 1, _s2.idx - 1) AS _w - FROM _q_0 AS _q_0 - CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) - CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx), LATERAL UNNEST(STRING_TO_ARRAY(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) + _s2.idx - 1 AS idx2, + _s4.idx - 1 AS idx3, + _s1.key, + _s4.val AS val3, + ROW_NUMBER() OVER (PARTITION BY _s1.key ORDER BY _s0.idx - 1 DESC, _s2.idx - 1, _s4.idx - 1) AS _w + FROM _s1 AS _s1 + CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s1.comment, '.')) WITH ORDINALITY AS _s0(val, idx) + CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s0.val, ',')) WITH ORDINALITY AS _s2(val, idx), LATERAL UNNEST(STRING_TO_ARRAY(_s2.val, ' ')) WITH ORDINALITY AS _s4(val, idx) WHERE - _s2.val <> '' + _s4.val <> '' ) SELECT key, diff --git a/tests/test_sql_refsols/explode_15_snowflake.sql b/tests/test_sql_refsols/explode_15_snowflake.sql index aa51f1fff..815f6b4cf 100644 --- a/tests/test_sql_refsols/explode_15_snowflake.sql +++ b/tests/test_sql_refsols/explode_15_snowflake.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_comment AS comment, c_custkey AS key @@ -9,17 +9,17 @@ WITH _q_0 AS ( ), _t0 AS ( SELECT _s0.index - 1 AS idx1, - _s1.index - 1 AS idx2, - _s2.index - 1 AS idx3, - _q_0.key, - _s2.value AS val3 - FROM _q_0 AS _q_0 - CROSS JOIN LATERAL SPLIT_TO_TABLE(_q_0.comment, '.') AS _s0 - CROSS JOIN LATERAL SPLIT_TO_TABLE(_s0.value, ',') AS _s1, LATERAL SPLIT_TO_TABLE(_s1.value, ' ') AS _s2 + _s2.index - 1 AS idx2, + _s4.index - 1 AS idx3, + _s1.key, + _s4.value AS val3 + FROM _s1 AS _s1 + CROSS JOIN LATERAL SPLIT_TO_TABLE(_s1.comment, '.') AS _s0 + CROSS JOIN LATERAL SPLIT_TO_TABLE(_s0.value, ',') AS _s2, LATERAL SPLIT_TO_TABLE(_s2.value, ' ') AS _s4 WHERE - _s2.value <> '' + _s4.value <> '' QUALIFY - ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.index - 1 DESC, _s1.index - 1, _s2.index - 1) = 1 + ROW_NUMBER() OVER (PARTITION BY _s1.key ORDER BY _s0.index - 1 DESC, _s2.index - 1, _s4.index - 1) = 1 ) SELECT key, diff --git a/tests/test_sql_refsols/explode_15_trino.sql b/tests/test_sql_refsols/explode_15_trino.sql index e5767fb89..19d0a5e55 100644 --- a/tests/test_sql_refsols/explode_15_trino.sql +++ b/tests/test_sql_refsols/explode_15_trino.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_comment AS comment, c_custkey AS key @@ -9,16 +9,16 @@ WITH _q_0 AS ( ), _t AS ( SELECT _s0.idx - 1 AS idx1, - _s1.idx - 1 AS idx2, - _s2.idx - 1 AS idx3, - _q_0.key, - _s2.val AS val3, - ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.idx - 1 DESC NULLS FIRST, _s1.idx - 1, _s2.idx - 1) AS _w - FROM _q_0 AS _q_0 - CROSS JOIN UNNEST(SPLIT(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) - CROSS JOIN UNNEST(SPLIT(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx), UNNEST(SPLIT(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) + _s2.idx - 1 AS idx2, + _s4.idx - 1 AS idx3, + _s1.key, + _s4.val AS val3, + ROW_NUMBER() OVER (PARTITION BY _s1.key ORDER BY _s0.idx - 1 DESC NULLS FIRST, _s2.idx - 1, _s4.idx - 1) AS _w + FROM _s1 AS _s1 + CROSS JOIN UNNEST(SPLIT(_s1.comment, '.')) WITH ORDINALITY AS _s0(val, idx) + CROSS JOIN UNNEST(SPLIT(_s0.val, ',')) WITH ORDINALITY AS _s2(val, idx), UNNEST(SPLIT(_s2.val, ' ')) WITH ORDINALITY AS _s4(val, idx) WHERE - _s2.val <> '' + _s4.val <> '' ) SELECT key, diff --git a/tests/test_sql_refsols/explode_16_databricks.sql b/tests/test_sql_refsols/explode_16_databricks.sql index 07d9380af..e0f9ee103 100644 --- a/tests/test_sql_refsols/explode_16_databricks.sql +++ b/tests/test_sql_refsols/explode_16_databricks.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_comment AS comment, c_custkey AS key @@ -9,17 +9,17 @@ WITH _q_0 AS ( ), _t0 AS ( SELECT _s0.idx AS idx1, - _s1.idx AS idx2, - _s2.idx AS idx3, - _q_0.key, - _s2.val AS val3 - FROM _q_0 AS _q_0 - CROSS JOIN LATERAL POSEXPLODE(SPLIT(_q_0.comment, '\\Q.\\E')) AS _s0(idx, val) - CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s0.val, '\\Q,\\E')) AS _s1(idx, val), LATERAL POSEXPLODE(SPLIT(_s1.val, '\\Q \\E')) AS _s2(idx, val) + _s2.idx AS idx2, + _s4.idx AS idx3, + _s1.key, + _s4.val AS val3 + FROM _s1 AS _s1 + CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s1.comment, '\\Q.\\E')) AS _s0(idx, val) + CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s0.val, '\\Q,\\E')) AS _s2(idx, val), LATERAL POSEXPLODE(SPLIT(_s2.val, '\\Q \\E')) AS _s4(idx, val) WHERE - _s2.val <> '' + _s4.val <> '' QUALIFY - ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.idx NULLS LAST, _s1.idx DESC NULLS FIRST, _s2.idx NULLS LAST) = 1 + ROW_NUMBER() OVER (PARTITION BY _s1.key ORDER BY _s0.idx NULLS LAST, _s2.idx DESC NULLS FIRST, _s4.idx NULLS LAST) = 1 ) SELECT key, diff --git a/tests/test_sql_refsols/explode_16_duckdb.sql b/tests/test_sql_refsols/explode_16_duckdb.sql index 2f5d6ce5d..965998ded 100644 --- a/tests/test_sql_refsols/explode_16_duckdb.sql +++ b/tests/test_sql_refsols/explode_16_duckdb.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_comment AS comment, c_custkey AS key @@ -9,29 +9,29 @@ WITH _q_0 AS ( ), _t0 AS ( SELECT _s0.idx AS idx1, - _s1.idx AS idx2, - _s2.idx AS idx3, - _q_0.key, - _s2.val AS val3 - FROM _q_0 AS _q_0 + _s2.idx AS idx2, + _s4.idx AS idx3, + _s1.key, + _s4.val AS val3 + FROM _s1 AS _s1 CROSS JOIN LATERAL ( SELECT - UNNEST(STR_SPLIT(_q_0.comment, '.')) AS _col_0, - GENERATE_SUBSCRIPTS(STR_SPLIT(_q_0.comment, '.'), 1) - 1 AS _col_1 + UNNEST(STR_SPLIT(_s1.comment, '.')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_s1.comment, '.'), 1) - 1 AS _col_1 ) AS _s0(val, idx) CROSS JOIN LATERAL ( SELECT UNNEST(STR_SPLIT(_s0.val, ',')) AS _col_0, GENERATE_SUBSCRIPTS(STR_SPLIT(_s0.val, ','), 1) - 1 AS _col_1 - ) AS _s1(val, idx), LATERAL ( + ) AS _s2(val, idx), LATERAL ( SELECT - UNNEST(STR_SPLIT(_s1.val, ' ')) AS _col_0, - GENERATE_SUBSCRIPTS(STR_SPLIT(_s1.val, ' '), 1) - 1 AS _col_1 - ) AS _s2(val, idx) + UNNEST(STR_SPLIT(_s2.val, ' ')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_s2.val, ' '), 1) - 1 AS _col_1 + ) AS _s4(val, idx) WHERE - _s2.val <> '' + _s4.val <> '' QUALIFY - ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.idx, _s1.idx DESC NULLS FIRST, _s2.idx) = 1 + ROW_NUMBER() OVER (PARTITION BY _s1.key ORDER BY _s0.idx, _s2.idx DESC NULLS FIRST, _s4.idx) = 1 ) SELECT key, diff --git a/tests/test_sql_refsols/explode_16_postgres.sql b/tests/test_sql_refsols/explode_16_postgres.sql index 171e508fe..e483994af 100644 --- a/tests/test_sql_refsols/explode_16_postgres.sql +++ b/tests/test_sql_refsols/explode_16_postgres.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_comment AS comment, c_custkey AS key @@ -9,16 +9,16 @@ WITH _q_0 AS ( ), _t AS ( SELECT _s0.idx - 1 AS idx1, - _s1.idx - 1 AS idx2, - _s2.idx - 1 AS idx3, - _q_0.key, - _s2.val AS val3, - ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.idx - 1, _s1.idx - 1 DESC, _s2.idx - 1) AS _w - FROM _q_0 AS _q_0 - CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) - CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx), LATERAL UNNEST(STRING_TO_ARRAY(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) + _s2.idx - 1 AS idx2, + _s4.idx - 1 AS idx3, + _s1.key, + _s4.val AS val3, + ROW_NUMBER() OVER (PARTITION BY _s1.key ORDER BY _s0.idx - 1, _s2.idx - 1 DESC, _s4.idx - 1) AS _w + FROM _s1 AS _s1 + CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s1.comment, '.')) WITH ORDINALITY AS _s0(val, idx) + CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s0.val, ',')) WITH ORDINALITY AS _s2(val, idx), LATERAL UNNEST(STRING_TO_ARRAY(_s2.val, ' ')) WITH ORDINALITY AS _s4(val, idx) WHERE - _s2.val <> '' + _s4.val <> '' ) SELECT key, diff --git a/tests/test_sql_refsols/explode_16_snowflake.sql b/tests/test_sql_refsols/explode_16_snowflake.sql index 410f28645..2306f3c31 100644 --- a/tests/test_sql_refsols/explode_16_snowflake.sql +++ b/tests/test_sql_refsols/explode_16_snowflake.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_comment AS comment, c_custkey AS key @@ -9,17 +9,17 @@ WITH _q_0 AS ( ), _t0 AS ( SELECT _s0.index - 1 AS idx1, - _s1.index - 1 AS idx2, - _s2.index - 1 AS idx3, - _q_0.key, - _s2.value AS val3 - FROM _q_0 AS _q_0 - CROSS JOIN LATERAL SPLIT_TO_TABLE(_q_0.comment, '.') AS _s0 - CROSS JOIN LATERAL SPLIT_TO_TABLE(_s0.value, ',') AS _s1, LATERAL SPLIT_TO_TABLE(_s1.value, ' ') AS _s2 + _s2.index - 1 AS idx2, + _s4.index - 1 AS idx3, + _s1.key, + _s4.value AS val3 + FROM _s1 AS _s1 + CROSS JOIN LATERAL SPLIT_TO_TABLE(_s1.comment, '.') AS _s0 + CROSS JOIN LATERAL SPLIT_TO_TABLE(_s0.value, ',') AS _s2, LATERAL SPLIT_TO_TABLE(_s2.value, ' ') AS _s4 WHERE - _s2.value <> '' + _s4.value <> '' QUALIFY - ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.index - 1, _s1.index - 1 DESC, _s2.index - 1) = 1 + ROW_NUMBER() OVER (PARTITION BY _s1.key ORDER BY _s0.index - 1, _s2.index - 1 DESC, _s4.index - 1) = 1 ) SELECT key, diff --git a/tests/test_sql_refsols/explode_16_trino.sql b/tests/test_sql_refsols/explode_16_trino.sql index abc405f1a..85c56f419 100644 --- a/tests/test_sql_refsols/explode_16_trino.sql +++ b/tests/test_sql_refsols/explode_16_trino.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_comment AS comment, c_custkey AS key @@ -9,16 +9,16 @@ WITH _q_0 AS ( ), _t AS ( SELECT _s0.idx - 1 AS idx1, - _s1.idx - 1 AS idx2, - _s2.idx - 1 AS idx3, - _q_0.key, - _s2.val AS val3, - ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.idx - 1, _s1.idx - 1 DESC NULLS FIRST, _s2.idx - 1) AS _w - FROM _q_0 AS _q_0 - CROSS JOIN UNNEST(SPLIT(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) - CROSS JOIN UNNEST(SPLIT(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx), UNNEST(SPLIT(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) + _s2.idx - 1 AS idx2, + _s4.idx - 1 AS idx3, + _s1.key, + _s4.val AS val3, + ROW_NUMBER() OVER (PARTITION BY _s1.key ORDER BY _s0.idx - 1, _s2.idx - 1 DESC NULLS FIRST, _s4.idx - 1) AS _w + FROM _s1 AS _s1 + CROSS JOIN UNNEST(SPLIT(_s1.comment, '.')) WITH ORDINALITY AS _s0(val, idx) + CROSS JOIN UNNEST(SPLIT(_s0.val, ',')) WITH ORDINALITY AS _s2(val, idx), UNNEST(SPLIT(_s2.val, ' ')) WITH ORDINALITY AS _s4(val, idx) WHERE - _s2.val <> '' + _s4.val <> '' ) SELECT key, diff --git a/tests/test_sql_refsols/explode_17_databricks.sql b/tests/test_sql_refsols/explode_17_databricks.sql index b49c2afaf..e9fae26c9 100644 --- a/tests/test_sql_refsols/explode_17_databricks.sql +++ b/tests/test_sql_refsols/explode_17_databricks.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_comment AS comment, c_custkey AS key @@ -9,17 +9,17 @@ WITH _q_0 AS ( ), _t0 AS ( SELECT _s0.idx AS idx1, - _s1.idx AS idx2, - _s2.idx AS idx3, - _q_0.key, - _s2.val AS val3 - FROM _q_0 AS _q_0 - CROSS JOIN LATERAL POSEXPLODE(SPLIT(_q_0.comment, '\\Q.\\E')) AS _s0(idx, val) - CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s0.val, '\\Q,\\E')) AS _s1(idx, val), LATERAL POSEXPLODE(SPLIT(_s1.val, '\\Q \\E')) AS _s2(idx, val) + _s2.idx AS idx2, + _s4.idx AS idx3, + _s1.key, + _s4.val AS val3 + FROM _s1 AS _s1 + CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s1.comment, '\\Q.\\E')) AS _s0(idx, val) + CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s0.val, '\\Q,\\E')) AS _s2(idx, val), LATERAL POSEXPLODE(SPLIT(_s2.val, '\\Q \\E')) AS _s4(idx, val) WHERE - _s2.val <> '' + _s4.val <> '' QUALIFY - ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.idx DESC NULLS FIRST, _s1.idx DESC NULLS FIRST, _s2.idx NULLS LAST) = 1 + ROW_NUMBER() OVER (PARTITION BY _s1.key ORDER BY _s0.idx DESC NULLS FIRST, _s2.idx DESC NULLS FIRST, _s4.idx NULLS LAST) = 1 ) SELECT key, diff --git a/tests/test_sql_refsols/explode_17_duckdb.sql b/tests/test_sql_refsols/explode_17_duckdb.sql index d3c268319..6d1fae2f3 100644 --- a/tests/test_sql_refsols/explode_17_duckdb.sql +++ b/tests/test_sql_refsols/explode_17_duckdb.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_comment AS comment, c_custkey AS key @@ -9,29 +9,29 @@ WITH _q_0 AS ( ), _t0 AS ( SELECT _s0.idx AS idx1, - _s1.idx AS idx2, - _s2.idx AS idx3, - _q_0.key, - _s2.val AS val3 - FROM _q_0 AS _q_0 + _s2.idx AS idx2, + _s4.idx AS idx3, + _s1.key, + _s4.val AS val3 + FROM _s1 AS _s1 CROSS JOIN LATERAL ( SELECT - UNNEST(STR_SPLIT(_q_0.comment, '.')) AS _col_0, - GENERATE_SUBSCRIPTS(STR_SPLIT(_q_0.comment, '.'), 1) - 1 AS _col_1 + UNNEST(STR_SPLIT(_s1.comment, '.')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_s1.comment, '.'), 1) - 1 AS _col_1 ) AS _s0(val, idx) CROSS JOIN LATERAL ( SELECT UNNEST(STR_SPLIT(_s0.val, ',')) AS _col_0, GENERATE_SUBSCRIPTS(STR_SPLIT(_s0.val, ','), 1) - 1 AS _col_1 - ) AS _s1(val, idx), LATERAL ( + ) AS _s2(val, idx), LATERAL ( SELECT - UNNEST(STR_SPLIT(_s1.val, ' ')) AS _col_0, - GENERATE_SUBSCRIPTS(STR_SPLIT(_s1.val, ' '), 1) - 1 AS _col_1 - ) AS _s2(val, idx) + UNNEST(STR_SPLIT(_s2.val, ' ')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_s2.val, ' '), 1) - 1 AS _col_1 + ) AS _s4(val, idx) WHERE - _s2.val <> '' + _s4.val <> '' QUALIFY - ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.idx DESC NULLS FIRST, _s1.idx DESC NULLS FIRST, _s2.idx) = 1 + ROW_NUMBER() OVER (PARTITION BY _s1.key ORDER BY _s0.idx DESC NULLS FIRST, _s2.idx DESC NULLS FIRST, _s4.idx) = 1 ) SELECT key, diff --git a/tests/test_sql_refsols/explode_17_postgres.sql b/tests/test_sql_refsols/explode_17_postgres.sql index 01c4d2c92..16e4b3566 100644 --- a/tests/test_sql_refsols/explode_17_postgres.sql +++ b/tests/test_sql_refsols/explode_17_postgres.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_comment AS comment, c_custkey AS key @@ -9,16 +9,16 @@ WITH _q_0 AS ( ), _t AS ( SELECT _s0.idx - 1 AS idx1, - _s1.idx - 1 AS idx2, - _s2.idx - 1 AS idx3, - _q_0.key, - _s2.val AS val3, - ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.idx - 1 DESC, _s1.idx - 1 DESC, _s2.idx - 1) AS _w - FROM _q_0 AS _q_0 - CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) - CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx), LATERAL UNNEST(STRING_TO_ARRAY(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) + _s2.idx - 1 AS idx2, + _s4.idx - 1 AS idx3, + _s1.key, + _s4.val AS val3, + ROW_NUMBER() OVER (PARTITION BY _s1.key ORDER BY _s0.idx - 1 DESC, _s2.idx - 1 DESC, _s4.idx - 1) AS _w + FROM _s1 AS _s1 + CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s1.comment, '.')) WITH ORDINALITY AS _s0(val, idx) + CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s0.val, ',')) WITH ORDINALITY AS _s2(val, idx), LATERAL UNNEST(STRING_TO_ARRAY(_s2.val, ' ')) WITH ORDINALITY AS _s4(val, idx) WHERE - _s2.val <> '' + _s4.val <> '' ) SELECT key, diff --git a/tests/test_sql_refsols/explode_17_snowflake.sql b/tests/test_sql_refsols/explode_17_snowflake.sql index 623acd389..9bd4e8164 100644 --- a/tests/test_sql_refsols/explode_17_snowflake.sql +++ b/tests/test_sql_refsols/explode_17_snowflake.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_comment AS comment, c_custkey AS key @@ -9,17 +9,17 @@ WITH _q_0 AS ( ), _t0 AS ( SELECT _s0.index - 1 AS idx1, - _s1.index - 1 AS idx2, - _s2.index - 1 AS idx3, - _q_0.key, - _s2.value AS val3 - FROM _q_0 AS _q_0 - CROSS JOIN LATERAL SPLIT_TO_TABLE(_q_0.comment, '.') AS _s0 - CROSS JOIN LATERAL SPLIT_TO_TABLE(_s0.value, ',') AS _s1, LATERAL SPLIT_TO_TABLE(_s1.value, ' ') AS _s2 + _s2.index - 1 AS idx2, + _s4.index - 1 AS idx3, + _s1.key, + _s4.value AS val3 + FROM _s1 AS _s1 + CROSS JOIN LATERAL SPLIT_TO_TABLE(_s1.comment, '.') AS _s0 + CROSS JOIN LATERAL SPLIT_TO_TABLE(_s0.value, ',') AS _s2, LATERAL SPLIT_TO_TABLE(_s2.value, ' ') AS _s4 WHERE - _s2.value <> '' + _s4.value <> '' QUALIFY - ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.index - 1 DESC, _s1.index - 1 DESC, _s2.index - 1) = 1 + ROW_NUMBER() OVER (PARTITION BY _s1.key ORDER BY _s0.index - 1 DESC, _s2.index - 1 DESC, _s4.index - 1) = 1 ) SELECT key, diff --git a/tests/test_sql_refsols/explode_17_trino.sql b/tests/test_sql_refsols/explode_17_trino.sql index 83e26edb0..c4415234c 100644 --- a/tests/test_sql_refsols/explode_17_trino.sql +++ b/tests/test_sql_refsols/explode_17_trino.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_comment AS comment, c_custkey AS key @@ -9,16 +9,16 @@ WITH _q_0 AS ( ), _t AS ( SELECT _s0.idx - 1 AS idx1, - _s1.idx - 1 AS idx2, - _s2.idx - 1 AS idx3, - _q_0.key, - _s2.val AS val3, - ROW_NUMBER() OVER (PARTITION BY _q_0.key ORDER BY _s0.idx - 1 DESC NULLS FIRST, _s1.idx - 1 DESC NULLS FIRST, _s2.idx - 1) AS _w - FROM _q_0 AS _q_0 - CROSS JOIN UNNEST(SPLIT(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) - CROSS JOIN UNNEST(SPLIT(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx), UNNEST(SPLIT(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) + _s2.idx - 1 AS idx2, + _s4.idx - 1 AS idx3, + _s1.key, + _s4.val AS val3, + ROW_NUMBER() OVER (PARTITION BY _s1.key ORDER BY _s0.idx - 1 DESC NULLS FIRST, _s2.idx - 1 DESC NULLS FIRST, _s4.idx - 1) AS _w + FROM _s1 AS _s1 + CROSS JOIN UNNEST(SPLIT(_s1.comment, '.')) WITH ORDINALITY AS _s0(val, idx) + CROSS JOIN UNNEST(SPLIT(_s0.val, ',')) WITH ORDINALITY AS _s2(val, idx), UNNEST(SPLIT(_s2.val, ' ')) WITH ORDINALITY AS _s4(val, idx) WHERE - _s2.val <> '' + _s4.val <> '' ) SELECT key, diff --git a/tests/test_sql_refsols/explode_18_databricks.sql b/tests/test_sql_refsols/explode_18_databricks.sql index 700c8264d..e5c5e772c 100644 --- a/tests/test_sql_refsols/explode_18_databricks.sql +++ b/tests/test_sql_refsols/explode_18_databricks.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_comment AS comment, c_custkey AS key @@ -9,17 +9,17 @@ WITH _q_0 AS ( ), _t0 AS ( SELECT _s0.idx AS idx1, - _s1.idx AS idx2, - _s2.idx AS idx3, - _q_0.key, - _s2.val AS val3 - FROM _q_0 AS _q_0 - CROSS JOIN LATERAL POSEXPLODE(SPLIT(_q_0.comment, '\\Q.\\E')) AS _s0(idx, val) - CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s0.val, '\\Q,\\E')) AS _s1(idx, val), LATERAL POSEXPLODE(SPLIT(_s1.val, '\\Q \\E')) AS _s2(idx, val) + _s2.idx AS idx2, + _s4.idx AS idx3, + _s1.key, + _s4.val AS val3 + FROM _s1 AS _s1 + CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s1.comment, '\\Q.\\E')) AS _s0(idx, val) + CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s0.val, '\\Q,\\E')) AS _s2(idx, val), LATERAL POSEXPLODE(SPLIT(_s2.val, '\\Q \\E')) AS _s4(idx, val) WHERE - _s2.val <> '' + _s4.val <> '' QUALIFY - ROW_NUMBER() OVER (PARTITION BY _s0.idx, _q_0.key, _s1.idx ORDER BY _s2.idx DESC NULLS FIRST) = 1 + ROW_NUMBER() OVER (PARTITION BY _s0.idx, _s1.key, _s2.idx ORDER BY _s4.idx DESC NULLS FIRST) = 1 ) SELECT key, diff --git a/tests/test_sql_refsols/explode_18_duckdb.sql b/tests/test_sql_refsols/explode_18_duckdb.sql index 582ae1ca5..46d0a7c78 100644 --- a/tests/test_sql_refsols/explode_18_duckdb.sql +++ b/tests/test_sql_refsols/explode_18_duckdb.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_comment AS comment, c_custkey AS key @@ -9,29 +9,29 @@ WITH _q_0 AS ( ), _t0 AS ( SELECT _s0.idx AS idx1, - _s1.idx AS idx2, - _s2.idx AS idx3, - _q_0.key, - _s2.val AS val3 - FROM _q_0 AS _q_0 + _s2.idx AS idx2, + _s4.idx AS idx3, + _s1.key, + _s4.val AS val3 + FROM _s1 AS _s1 CROSS JOIN LATERAL ( SELECT - UNNEST(STR_SPLIT(_q_0.comment, '.')) AS _col_0, - GENERATE_SUBSCRIPTS(STR_SPLIT(_q_0.comment, '.'), 1) - 1 AS _col_1 + UNNEST(STR_SPLIT(_s1.comment, '.')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_s1.comment, '.'), 1) - 1 AS _col_1 ) AS _s0(val, idx) CROSS JOIN LATERAL ( SELECT UNNEST(STR_SPLIT(_s0.val, ',')) AS _col_0, GENERATE_SUBSCRIPTS(STR_SPLIT(_s0.val, ','), 1) - 1 AS _col_1 - ) AS _s1(val, idx), LATERAL ( + ) AS _s2(val, idx), LATERAL ( SELECT - UNNEST(STR_SPLIT(_s1.val, ' ')) AS _col_0, - GENERATE_SUBSCRIPTS(STR_SPLIT(_s1.val, ' '), 1) - 1 AS _col_1 - ) AS _s2(val, idx) + UNNEST(STR_SPLIT(_s2.val, ' ')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_s2.val, ' '), 1) - 1 AS _col_1 + ) AS _s4(val, idx) WHERE - _s2.val <> '' + _s4.val <> '' QUALIFY - ROW_NUMBER() OVER (PARTITION BY _s0.idx, _q_0.key, _s1.idx ORDER BY _s2.idx DESC NULLS FIRST) = 1 + ROW_NUMBER() OVER (PARTITION BY _s0.idx, _s1.key, _s2.idx ORDER BY _s4.idx DESC NULLS FIRST) = 1 ) SELECT key, diff --git a/tests/test_sql_refsols/explode_18_postgres.sql b/tests/test_sql_refsols/explode_18_postgres.sql index a6e221595..abf837b24 100644 --- a/tests/test_sql_refsols/explode_18_postgres.sql +++ b/tests/test_sql_refsols/explode_18_postgres.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_comment AS comment, c_custkey AS key @@ -9,16 +9,16 @@ WITH _q_0 AS ( ), _t AS ( SELECT _s0.idx - 1 AS idx1, - _s1.idx - 1 AS idx2, - _s2.idx - 1 AS idx3, - _q_0.key, - _s2.val AS val3, - ROW_NUMBER() OVER (PARTITION BY _s0.idx - 1, _q_0.key, _s1.idx - 1 ORDER BY _s2.idx - 1 DESC) AS _w - FROM _q_0 AS _q_0 - CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) - CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx), LATERAL UNNEST(STRING_TO_ARRAY(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) + _s2.idx - 1 AS idx2, + _s4.idx - 1 AS idx3, + _s1.key, + _s4.val AS val3, + ROW_NUMBER() OVER (PARTITION BY _s0.idx - 1, _s1.key, _s2.idx - 1 ORDER BY _s4.idx - 1 DESC) AS _w + FROM _s1 AS _s1 + CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s1.comment, '.')) WITH ORDINALITY AS _s0(val, idx) + CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s0.val, ',')) WITH ORDINALITY AS _s2(val, idx), LATERAL UNNEST(STRING_TO_ARRAY(_s2.val, ' ')) WITH ORDINALITY AS _s4(val, idx) WHERE - _s2.val <> '' + _s4.val <> '' ) SELECT key, diff --git a/tests/test_sql_refsols/explode_18_snowflake.sql b/tests/test_sql_refsols/explode_18_snowflake.sql index e652b0fa7..57b1e2049 100644 --- a/tests/test_sql_refsols/explode_18_snowflake.sql +++ b/tests/test_sql_refsols/explode_18_snowflake.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_comment AS comment, c_custkey AS key @@ -9,17 +9,17 @@ WITH _q_0 AS ( ), _t0 AS ( SELECT _s0.index - 1 AS idx1, - _s1.index - 1 AS idx2, - _s2.index - 1 AS idx3, - _q_0.key, - _s2.value AS val3 - FROM _q_0 AS _q_0 - CROSS JOIN LATERAL SPLIT_TO_TABLE(_q_0.comment, '.') AS _s0 - CROSS JOIN LATERAL SPLIT_TO_TABLE(_s0.value, ',') AS _s1, LATERAL SPLIT_TO_TABLE(_s1.value, ' ') AS _s2 + _s2.index - 1 AS idx2, + _s4.index - 1 AS idx3, + _s1.key, + _s4.value AS val3 + FROM _s1 AS _s1 + CROSS JOIN LATERAL SPLIT_TO_TABLE(_s1.comment, '.') AS _s0 + CROSS JOIN LATERAL SPLIT_TO_TABLE(_s0.value, ',') AS _s2, LATERAL SPLIT_TO_TABLE(_s2.value, ' ') AS _s4 WHERE - _s2.value <> '' + _s4.value <> '' QUALIFY - ROW_NUMBER() OVER (PARTITION BY _s0.index - 1, _q_0.key, _s1.index - 1 ORDER BY _s2.index - 1 DESC) = 1 + ROW_NUMBER() OVER (PARTITION BY _s0.index - 1, _s1.key, _s2.index - 1 ORDER BY _s4.index - 1 DESC) = 1 ) SELECT key, diff --git a/tests/test_sql_refsols/explode_18_trino.sql b/tests/test_sql_refsols/explode_18_trino.sql index 8a89f472f..09a023f92 100644 --- a/tests/test_sql_refsols/explode_18_trino.sql +++ b/tests/test_sql_refsols/explode_18_trino.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_comment AS comment, c_custkey AS key @@ -9,16 +9,16 @@ WITH _q_0 AS ( ), _t AS ( SELECT _s0.idx - 1 AS idx1, - _s1.idx - 1 AS idx2, - _s2.idx - 1 AS idx3, - _q_0.key, - _s2.val AS val3, - ROW_NUMBER() OVER (PARTITION BY _s0.idx - 1, _q_0.key, _s1.idx - 1 ORDER BY _s2.idx - 1 DESC NULLS FIRST) AS _w - FROM _q_0 AS _q_0 - CROSS JOIN UNNEST(SPLIT(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) - CROSS JOIN UNNEST(SPLIT(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx), UNNEST(SPLIT(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) + _s2.idx - 1 AS idx2, + _s4.idx - 1 AS idx3, + _s1.key, + _s4.val AS val3, + ROW_NUMBER() OVER (PARTITION BY _s0.idx - 1, _s1.key, _s2.idx - 1 ORDER BY _s4.idx - 1 DESC NULLS FIRST) AS _w + FROM _s1 AS _s1 + CROSS JOIN UNNEST(SPLIT(_s1.comment, '.')) WITH ORDINALITY AS _s0(val, idx) + CROSS JOIN UNNEST(SPLIT(_s0.val, ',')) WITH ORDINALITY AS _s2(val, idx), UNNEST(SPLIT(_s2.val, ' ')) WITH ORDINALITY AS _s4(val, idx) WHERE - _s2.val <> '' + _s4.val <> '' ) SELECT key, diff --git a/tests/test_sql_refsols/explode_19_databricks.sql b/tests/test_sql_refsols/explode_19_databricks.sql index fccdda8e9..1dc97da91 100644 --- a/tests/test_sql_refsols/explode_19_databricks.sql +++ b/tests/test_sql_refsols/explode_19_databricks.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_comment AS comment, c_custkey AS key @@ -9,17 +9,17 @@ WITH _q_0 AS ( ), _t0 AS ( SELECT _s0.idx AS idx1, - _s1.idx AS idx2, - _s2.idx AS idx3, - _q_0.key, - _s2.val AS val3 - FROM _q_0 AS _q_0 - CROSS JOIN LATERAL POSEXPLODE(SPLIT(_q_0.comment, '\\Q.\\E')) AS _s0(idx, val) - CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s0.val, '\\Q,\\E')) AS _s1(idx, val), LATERAL POSEXPLODE(SPLIT(_s1.val, '\\Q \\E')) AS _s2(idx, val) + _s2.idx AS idx2, + _s4.idx AS idx3, + _s1.key, + _s4.val AS val3 + FROM _s1 AS _s1 + CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s1.comment, '\\Q.\\E')) AS _s0(idx, val) + CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s0.val, '\\Q,\\E')) AS _s2(idx, val), LATERAL POSEXPLODE(SPLIT(_s2.val, '\\Q \\E')) AS _s4(idx, val) WHERE - _s2.val <> '' + _s4.val <> '' QUALIFY - ROW_NUMBER() OVER (PARTITION BY _q_0.key, _s0.idx ORDER BY _s1.idx DESC NULLS FIRST, _s2.idx DESC NULLS FIRST) = 1 + ROW_NUMBER() OVER (PARTITION BY _s1.key, _s0.idx ORDER BY _s2.idx DESC NULLS FIRST, _s4.idx DESC NULLS FIRST) = 1 ) SELECT key, diff --git a/tests/test_sql_refsols/explode_19_duckdb.sql b/tests/test_sql_refsols/explode_19_duckdb.sql index 701e3b0b3..c9e33fe97 100644 --- a/tests/test_sql_refsols/explode_19_duckdb.sql +++ b/tests/test_sql_refsols/explode_19_duckdb.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_comment AS comment, c_custkey AS key @@ -9,29 +9,29 @@ WITH _q_0 AS ( ), _t0 AS ( SELECT _s0.idx AS idx1, - _s1.idx AS idx2, - _s2.idx AS idx3, - _q_0.key, - _s2.val AS val3 - FROM _q_0 AS _q_0 + _s2.idx AS idx2, + _s4.idx AS idx3, + _s1.key, + _s4.val AS val3 + FROM _s1 AS _s1 CROSS JOIN LATERAL ( SELECT - UNNEST(STR_SPLIT(_q_0.comment, '.')) AS _col_0, - GENERATE_SUBSCRIPTS(STR_SPLIT(_q_0.comment, '.'), 1) - 1 AS _col_1 + UNNEST(STR_SPLIT(_s1.comment, '.')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_s1.comment, '.'), 1) - 1 AS _col_1 ) AS _s0(val, idx) CROSS JOIN LATERAL ( SELECT UNNEST(STR_SPLIT(_s0.val, ',')) AS _col_0, GENERATE_SUBSCRIPTS(STR_SPLIT(_s0.val, ','), 1) - 1 AS _col_1 - ) AS _s1(val, idx), LATERAL ( + ) AS _s2(val, idx), LATERAL ( SELECT - UNNEST(STR_SPLIT(_s1.val, ' ')) AS _col_0, - GENERATE_SUBSCRIPTS(STR_SPLIT(_s1.val, ' '), 1) - 1 AS _col_1 - ) AS _s2(val, idx) + UNNEST(STR_SPLIT(_s2.val, ' ')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_s2.val, ' '), 1) - 1 AS _col_1 + ) AS _s4(val, idx) WHERE - _s2.val <> '' + _s4.val <> '' QUALIFY - ROW_NUMBER() OVER (PARTITION BY _q_0.key, _s0.idx ORDER BY _s1.idx DESC NULLS FIRST, _s2.idx DESC NULLS FIRST) = 1 + ROW_NUMBER() OVER (PARTITION BY _s1.key, _s0.idx ORDER BY _s2.idx DESC NULLS FIRST, _s4.idx DESC NULLS FIRST) = 1 ) SELECT key, diff --git a/tests/test_sql_refsols/explode_19_postgres.sql b/tests/test_sql_refsols/explode_19_postgres.sql index 124ff40ff..c7e4d8fbe 100644 --- a/tests/test_sql_refsols/explode_19_postgres.sql +++ b/tests/test_sql_refsols/explode_19_postgres.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_comment AS comment, c_custkey AS key @@ -9,16 +9,16 @@ WITH _q_0 AS ( ), _t AS ( SELECT _s0.idx - 1 AS idx1, - _s1.idx - 1 AS idx2, - _s2.idx - 1 AS idx3, - _q_0.key, - _s2.val AS val3, - ROW_NUMBER() OVER (PARTITION BY _q_0.key, _s0.idx - 1 ORDER BY _s1.idx - 1 DESC, _s2.idx - 1 DESC) AS _w - FROM _q_0 AS _q_0 - CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) - CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx), LATERAL UNNEST(STRING_TO_ARRAY(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) + _s2.idx - 1 AS idx2, + _s4.idx - 1 AS idx3, + _s1.key, + _s4.val AS val3, + ROW_NUMBER() OVER (PARTITION BY _s1.key, _s0.idx - 1 ORDER BY _s2.idx - 1 DESC, _s4.idx - 1 DESC) AS _w + FROM _s1 AS _s1 + CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s1.comment, '.')) WITH ORDINALITY AS _s0(val, idx) + CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s0.val, ',')) WITH ORDINALITY AS _s2(val, idx), LATERAL UNNEST(STRING_TO_ARRAY(_s2.val, ' ')) WITH ORDINALITY AS _s4(val, idx) WHERE - _s2.val <> '' + _s4.val <> '' ) SELECT key, diff --git a/tests/test_sql_refsols/explode_19_snowflake.sql b/tests/test_sql_refsols/explode_19_snowflake.sql index 4d19779f9..16c9596f8 100644 --- a/tests/test_sql_refsols/explode_19_snowflake.sql +++ b/tests/test_sql_refsols/explode_19_snowflake.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_comment AS comment, c_custkey AS key @@ -9,17 +9,17 @@ WITH _q_0 AS ( ), _t0 AS ( SELECT _s0.index - 1 AS idx1, - _s1.index - 1 AS idx2, - _s2.index - 1 AS idx3, - _q_0.key, - _s2.value AS val3 - FROM _q_0 AS _q_0 - CROSS JOIN LATERAL SPLIT_TO_TABLE(_q_0.comment, '.') AS _s0 - CROSS JOIN LATERAL SPLIT_TO_TABLE(_s0.value, ',') AS _s1, LATERAL SPLIT_TO_TABLE(_s1.value, ' ') AS _s2 + _s2.index - 1 AS idx2, + _s4.index - 1 AS idx3, + _s1.key, + _s4.value AS val3 + FROM _s1 AS _s1 + CROSS JOIN LATERAL SPLIT_TO_TABLE(_s1.comment, '.') AS _s0 + CROSS JOIN LATERAL SPLIT_TO_TABLE(_s0.value, ',') AS _s2, LATERAL SPLIT_TO_TABLE(_s2.value, ' ') AS _s4 WHERE - _s2.value <> '' + _s4.value <> '' QUALIFY - ROW_NUMBER() OVER (PARTITION BY _q_0.key, _s0.index - 1 ORDER BY _s1.index - 1 DESC, _s2.index - 1 DESC) = 1 + ROW_NUMBER() OVER (PARTITION BY _s1.key, _s0.index - 1 ORDER BY _s2.index - 1 DESC, _s4.index - 1 DESC) = 1 ) SELECT key, diff --git a/tests/test_sql_refsols/explode_19_trino.sql b/tests/test_sql_refsols/explode_19_trino.sql index c7c2e0420..209ab4012 100644 --- a/tests/test_sql_refsols/explode_19_trino.sql +++ b/tests/test_sql_refsols/explode_19_trino.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_comment AS comment, c_custkey AS key @@ -9,16 +9,16 @@ WITH _q_0 AS ( ), _t AS ( SELECT _s0.idx - 1 AS idx1, - _s1.idx - 1 AS idx2, - _s2.idx - 1 AS idx3, - _q_0.key, - _s2.val AS val3, - ROW_NUMBER() OVER (PARTITION BY _q_0.key, _s0.idx - 1 ORDER BY _s1.idx - 1 DESC NULLS FIRST, _s2.idx - 1 DESC NULLS FIRST) AS _w - FROM _q_0 AS _q_0 - CROSS JOIN UNNEST(SPLIT(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) - CROSS JOIN UNNEST(SPLIT(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx), UNNEST(SPLIT(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx) + _s2.idx - 1 AS idx2, + _s4.idx - 1 AS idx3, + _s1.key, + _s4.val AS val3, + ROW_NUMBER() OVER (PARTITION BY _s1.key, _s0.idx - 1 ORDER BY _s2.idx - 1 DESC NULLS FIRST, _s4.idx - 1 DESC NULLS FIRST) AS _w + FROM _s1 AS _s1 + CROSS JOIN UNNEST(SPLIT(_s1.comment, '.')) WITH ORDINALITY AS _s0(val, idx) + CROSS JOIN UNNEST(SPLIT(_s0.val, ',')) WITH ORDINALITY AS _s2(val, idx), UNNEST(SPLIT(_s2.val, ' ')) WITH ORDINALITY AS _s4(val, idx) WHERE - _s2.val <> '' + _s4.val <> '' ) SELECT key, diff --git a/tests/test_sql_refsols/explode_20_databricks.sql b/tests/test_sql_refsols/explode_20_databricks.sql index 6ff5c740b..742f14fc2 100644 --- a/tests/test_sql_refsols/explode_20_databricks.sql +++ b/tests/test_sql_refsols/explode_20_databricks.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_comment AS comment FROM tpch.customer @@ -9,7 +9,7 @@ WITH _q_0 AS ( SELECT _s0.val AS char, COUNT(*) AS n -FROM _q_0 AS _q_0, LATERAL POSEXPLODE(SPLIT(_q_0.comment, '\\Q\\E')) AS _s0(idx, val) +FROM _s1 AS _s1, LATERAL POSEXPLODE(SPLIT(_s1.comment, '\\Q\\E')) AS _s0(idx, val) WHERE _s0.val <> '' GROUP BY diff --git a/tests/test_sql_refsols/explode_20_duckdb.sql b/tests/test_sql_refsols/explode_20_duckdb.sql index 6c3da3cff..051b16098 100644 --- a/tests/test_sql_refsols/explode_20_duckdb.sql +++ b/tests/test_sql_refsols/explode_20_duckdb.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_comment AS comment FROM tpch.customer @@ -9,10 +9,10 @@ WITH _q_0 AS ( SELECT _s0.val AS char, COUNT(*) AS n -FROM _q_0 AS _q_0, LATERAL ( +FROM _s1 AS _s1, LATERAL ( SELECT - UNNEST(REGEXP_SPLIT_TO_ARRAY(_q_0.comment, '')) AS _col_0, - GENERATE_SUBSCRIPTS(REGEXP_SPLIT_TO_ARRAY(_q_0.comment, ''), 1) - 1 AS _col_1 + UNNEST(REGEXP_SPLIT_TO_ARRAY(_s1.comment, '')) AS _col_0, + GENERATE_SUBSCRIPTS(REGEXP_SPLIT_TO_ARRAY(_s1.comment, ''), 1) - 1 AS _col_1 ) AS _s0(val, idx) WHERE _s0.val <> '' diff --git a/tests/test_sql_refsols/explode_20_postgres.sql b/tests/test_sql_refsols/explode_20_postgres.sql index 2320cea42..c42958c24 100644 --- a/tests/test_sql_refsols/explode_20_postgres.sql +++ b/tests/test_sql_refsols/explode_20_postgres.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_comment AS comment FROM tpch.customer @@ -9,7 +9,7 @@ WITH _q_0 AS ( SELECT _s0.val AS char, COUNT(*) AS n -FROM _q_0 AS _q_0, LATERAL UNNEST(REGEXP_SPLIT_TO_ARRAY(_q_0.comment, '')) WITH ORDINALITY AS _s0(val, idx) +FROM _s1 AS _s1, LATERAL UNNEST(REGEXP_SPLIT_TO_ARRAY(_s1.comment, '')) WITH ORDINALITY AS _s0(val, idx) WHERE _s0.val <> '' GROUP BY diff --git a/tests/test_sql_refsols/explode_20_snowflake.sql b/tests/test_sql_refsols/explode_20_snowflake.sql index 494f067ef..4dfa67c96 100644 --- a/tests/test_sql_refsols/explode_20_snowflake.sql +++ b/tests/test_sql_refsols/explode_20_snowflake.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_comment AS comment FROM tpch.customer @@ -9,7 +9,7 @@ WITH _q_0 AS ( SELECT _s0.value AS char, COUNT(*) AS n -FROM _q_0 AS _q_0, LATERAL FLATTEN(REGEXP_EXTRACT_ALL(_q_0.comment, '.{1}')) AS _s0(seq, key, path, index, value, this) +FROM _s1 AS _s1, LATERAL FLATTEN(REGEXP_EXTRACT_ALL(_s1.comment, '.{1}')) AS _s0(seq, key, path, index, value, this) WHERE _s0.value <> '' GROUP BY diff --git a/tests/test_sql_refsols/explode_20_trino.sql b/tests/test_sql_refsols/explode_20_trino.sql index a2363f843..8b6dbad5f 100644 --- a/tests/test_sql_refsols/explode_20_trino.sql +++ b/tests/test_sql_refsols/explode_20_trino.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_comment AS comment FROM tpch.customer @@ -9,7 +9,7 @@ WITH _q_0 AS ( SELECT _s0.val AS char, COUNT(*) AS n -FROM _q_0 AS _q_0, UNNEST(SPLIT(_q_0.comment, '')) WITH ORDINALITY AS _s0(val, idx) +FROM _s1 AS _s1, UNNEST(SPLIT(_s1.comment, '')) WITH ORDINALITY AS _s0(val, idx) WHERE _s0.val <> '' GROUP BY diff --git a/tests/test_sql_refsols/explode_21_databricks.sql b/tests/test_sql_refsols/explode_21_databricks.sql index 9115278ac..09f7f65f1 100644 --- a/tests/test_sql_refsols/explode_21_databricks.sql +++ b/tests/test_sql_refsols/explode_21_databricks.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_comment AS comment, c_custkey AS key @@ -9,19 +9,19 @@ WITH _q_0 AS ( ), _t0 AS ( SELECT _s0.idx AS idx1, - _s1.idx AS idx2, - _s2.idx AS idx3, - _s3.idx AS idx4, - _q_0.key, - _s3.val AS val4 - FROM _q_0 AS _q_0 - CROSS JOIN LATERAL POSEXPLODE(SPLIT(_q_0.comment, '\\Q.\\E')) AS _s0(idx, val) - CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s0.val, '\\Q,\\E')) AS _s1(idx, val) - CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s1.val, '\\Q \\E')) AS _s2(idx, val), LATERAL POSEXPLODE(SPLIT(_s2.val, '\\Q\\E')) AS _s3(idx, val) + _s2.idx AS idx2, + _s4.idx AS idx3, + _s6.idx AS idx4, + _s1.key, + _s6.val AS val4 + FROM _s1 AS _s1 + CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s1.comment, '\\Q.\\E')) AS _s0(idx, val) + CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s0.val, '\\Q,\\E')) AS _s2(idx, val) + CROSS JOIN LATERAL POSEXPLODE(SPLIT(_s2.val, '\\Q \\E')) AS _s4(idx, val), LATERAL POSEXPLODE(SPLIT(_s4.val, '\\Q\\E')) AS _s6(idx, val) WHERE - _s3.val <> '' + _s6.val <> '' QUALIFY - ROW_NUMBER() OVER (PARTITION BY _s1.idx, _s0.idx, _q_0.key, _s2.idx ORDER BY _s3.idx NULLS LAST) = 1 + ROW_NUMBER() OVER (PARTITION BY _s2.idx, _s0.idx, _s1.key, _s4.idx ORDER BY _s6.idx NULLS LAST) = 1 ) SELECT key, diff --git a/tests/test_sql_refsols/explode_21_duckdb.sql b/tests/test_sql_refsols/explode_21_duckdb.sql index 12f5d4ea6..4de618999 100644 --- a/tests/test_sql_refsols/explode_21_duckdb.sql +++ b/tests/test_sql_refsols/explode_21_duckdb.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_comment AS comment, c_custkey AS key @@ -9,35 +9,35 @@ WITH _q_0 AS ( ), _t0 AS ( SELECT _s0.idx AS idx1, - _s1.idx AS idx2, - _s2.idx AS idx3, - _s3.idx AS idx4, - _q_0.key, - _s3.val AS val4 - FROM _q_0 AS _q_0 + _s2.idx AS idx2, + _s4.idx AS idx3, + _s6.idx AS idx4, + _s1.key, + _s6.val AS val4 + FROM _s1 AS _s1 CROSS JOIN LATERAL ( SELECT - UNNEST(STR_SPLIT(_q_0.comment, '.')) AS _col_0, - GENERATE_SUBSCRIPTS(STR_SPLIT(_q_0.comment, '.'), 1) - 1 AS _col_1 + UNNEST(STR_SPLIT(_s1.comment, '.')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_s1.comment, '.'), 1) - 1 AS _col_1 ) AS _s0(val, idx) CROSS JOIN LATERAL ( SELECT UNNEST(STR_SPLIT(_s0.val, ',')) AS _col_0, GENERATE_SUBSCRIPTS(STR_SPLIT(_s0.val, ','), 1) - 1 AS _col_1 - ) AS _s1(val, idx) + ) AS _s2(val, idx) CROSS JOIN LATERAL ( SELECT - UNNEST(STR_SPLIT(_s1.val, ' ')) AS _col_0, - GENERATE_SUBSCRIPTS(STR_SPLIT(_s1.val, ' '), 1) - 1 AS _col_1 - ) AS _s2(val, idx), LATERAL ( + UNNEST(STR_SPLIT(_s2.val, ' ')) AS _col_0, + GENERATE_SUBSCRIPTS(STR_SPLIT(_s2.val, ' '), 1) - 1 AS _col_1 + ) AS _s4(val, idx), LATERAL ( SELECT - UNNEST(REGEXP_SPLIT_TO_ARRAY(_s2.val, '')) AS _col_0, - GENERATE_SUBSCRIPTS(REGEXP_SPLIT_TO_ARRAY(_s2.val, ''), 1) - 1 AS _col_1 - ) AS _s3(val, idx) + UNNEST(REGEXP_SPLIT_TO_ARRAY(_s4.val, '')) AS _col_0, + GENERATE_SUBSCRIPTS(REGEXP_SPLIT_TO_ARRAY(_s4.val, ''), 1) - 1 AS _col_1 + ) AS _s6(val, idx) WHERE - _s3.val <> '' + _s6.val <> '' QUALIFY - ROW_NUMBER() OVER (PARTITION BY _s1.idx, _s0.idx, _q_0.key, _s2.idx ORDER BY _s3.idx) = 1 + ROW_NUMBER() OVER (PARTITION BY _s2.idx, _s0.idx, _s1.key, _s4.idx ORDER BY _s6.idx) = 1 ) SELECT key, diff --git a/tests/test_sql_refsols/explode_21_postgres.sql b/tests/test_sql_refsols/explode_21_postgres.sql index d36d95c51..a4fa9562b 100644 --- a/tests/test_sql_refsols/explode_21_postgres.sql +++ b/tests/test_sql_refsols/explode_21_postgres.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_comment AS comment, c_custkey AS key @@ -9,18 +9,18 @@ WITH _q_0 AS ( ), _t AS ( SELECT _s0.idx - 1 AS idx1, - _s1.idx - 1 AS idx2, - _s2.idx - 1 AS idx3, - _s3.idx - 1 AS idx4, - _q_0.key, - _s3.val AS val4, - ROW_NUMBER() OVER (PARTITION BY _s1.idx - 1, _s0.idx - 1, _q_0.key, _s2.idx - 1 ORDER BY _s3.idx - 1) AS _w - FROM _q_0 AS _q_0 - CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) - CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx) - CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx), LATERAL UNNEST(REGEXP_SPLIT_TO_ARRAY(_s2.val, '')) WITH ORDINALITY AS _s3(val, idx) + _s2.idx - 1 AS idx2, + _s4.idx - 1 AS idx3, + _s6.idx - 1 AS idx4, + _s1.key, + _s6.val AS val4, + ROW_NUMBER() OVER (PARTITION BY _s2.idx - 1, _s0.idx - 1, _s1.key, _s4.idx - 1 ORDER BY _s6.idx - 1) AS _w + FROM _s1 AS _s1 + CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s1.comment, '.')) WITH ORDINALITY AS _s0(val, idx) + CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s0.val, ',')) WITH ORDINALITY AS _s2(val, idx) + CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(_s2.val, ' ')) WITH ORDINALITY AS _s4(val, idx), LATERAL UNNEST(REGEXP_SPLIT_TO_ARRAY(_s4.val, '')) WITH ORDINALITY AS _s6(val, idx) WHERE - _s3.val <> '' + _s6.val <> '' ) SELECT key, diff --git a/tests/test_sql_refsols/explode_21_snowflake.sql b/tests/test_sql_refsols/explode_21_snowflake.sql index 6fe6ffe64..53174dd4e 100644 --- a/tests/test_sql_refsols/explode_21_snowflake.sql +++ b/tests/test_sql_refsols/explode_21_snowflake.sql @@ -1,26 +1,27 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT - c_comment AS comment + c_comment AS comment, + c_custkey AS key FROM tpch.customer ORDER BY - c_custkey NULLS FIRST + 2 NULLS FIRST LIMIT 3 ), _t0 AS ( SELECT _s0.index - 1 AS idx1, - _s1.index - 1 AS idx2, - _s2.index - 1 AS idx3, - _s3.index - 1 AS idx4, - key, - _s3.value AS val4 - FROM _q_0 AS _q_0 - CROSS JOIN LATERAL SPLIT_TO_TABLE(_q_0.comment, '.') AS _s0 - CROSS JOIN LATERAL SPLIT_TO_TABLE(_s0.value, ',') AS _s1 - CROSS JOIN LATERAL SPLIT_TO_TABLE(_s1.value, ' ') AS _s2, LATERAL FLATTEN(REGEXP_EXTRACT_ALL(_s2.value, '.{1}')) AS _s3(seq, key, path, index, value, this) + _s2.index - 1 AS idx2, + _s4.index - 1 AS idx3, + _s6.index AS idx4, + _s1.key, + _s6.value AS val4 + FROM _s1 AS _s1 + CROSS JOIN LATERAL SPLIT_TO_TABLE(_s1.comment, '.') AS _s0 + CROSS JOIN LATERAL SPLIT_TO_TABLE(_s0.value, ',') AS _s2 + CROSS JOIN LATERAL SPLIT_TO_TABLE(_s2.value, ' ') AS _s4, LATERAL FLATTEN(REGEXP_EXTRACT_ALL(_s4.value, '.{1}')) AS _s6(seq, key, path, index, value, this) WHERE - _s3.value <> '' + _s6.value <> '' QUALIFY - ROW_NUMBER() OVER (PARTITION BY _s1.index - 1, _s0.index - 1, key, _s2.index - 1 ORDER BY _s3.index - 1) = 1 + ROW_NUMBER() OVER (PARTITION BY _s2.index - 1, _s0.index - 1, _s1.key, _s4.index - 1 ORDER BY _s6.index) = 1 ) SELECT key, diff --git a/tests/test_sql_refsols/explode_21_trino.sql b/tests/test_sql_refsols/explode_21_trino.sql index 352dae69f..e7da4f3ad 100644 --- a/tests/test_sql_refsols/explode_21_trino.sql +++ b/tests/test_sql_refsols/explode_21_trino.sql @@ -1,4 +1,4 @@ -WITH _q_0 AS ( +WITH _s1 AS ( SELECT c_comment AS comment, c_custkey AS key @@ -9,18 +9,18 @@ WITH _q_0 AS ( ), _t AS ( SELECT _s0.idx - 1 AS idx1, - _s1.idx - 1 AS idx2, - _s2.idx - 1 AS idx3, - _s3.idx - 1 AS idx4, - _q_0.key, - _s3.val AS val4, - ROW_NUMBER() OVER (PARTITION BY _s1.idx - 1, _s0.idx - 1, _q_0.key, _s2.idx - 1 ORDER BY _s3.idx - 1) AS _w - FROM _q_0 AS _q_0 - CROSS JOIN UNNEST(SPLIT(_q_0.comment, '.')) WITH ORDINALITY AS _s0(val, idx) - CROSS JOIN UNNEST(SPLIT(_s0.val, ',')) WITH ORDINALITY AS _s1(val, idx) - CROSS JOIN UNNEST(SPLIT(_s1.val, ' ')) WITH ORDINALITY AS _s2(val, idx), UNNEST(SPLIT(_s2.val, '')) WITH ORDINALITY AS _s3(val, idx) + _s2.idx - 1 AS idx2, + _s4.idx - 1 AS idx3, + _s6.idx - 1 AS idx4, + _s1.key, + _s6.val AS val4, + ROW_NUMBER() OVER (PARTITION BY _s2.idx - 1, _s0.idx - 1, _s1.key, _s4.idx - 1 ORDER BY _s6.idx - 1) AS _w + FROM _s1 AS _s1 + CROSS JOIN UNNEST(SPLIT(_s1.comment, '.')) WITH ORDINALITY AS _s0(val, idx) + CROSS JOIN UNNEST(SPLIT(_s0.val, ',')) WITH ORDINALITY AS _s2(val, idx) + CROSS JOIN UNNEST(SPLIT(_s2.val, ' ')) WITH ORDINALITY AS _s4(val, idx), UNNEST(SPLIT(_s4.val, '')) WITH ORDINALITY AS _s6(val, idx) WHERE - _s3.val <> '' + _s6.val <> '' ) SELECT key, From 51517f684a93b9bb651d8cb762c0495607df5c02 Mon Sep 17 00:00:00 2001 From: knassre-bodo Date: Thu, 20 Aug 2026 00:01:37 -0700 Subject: [PATCH 32/44] [RUN CI][RUN DIALECTS] From 069ad2e2bd18c981b08bd777b3e8abc34feab1c4 Mon Sep 17 00:00:00 2001 From: knassre-bodo Date: Thu, 20 Aug 2026 10:14:28 -0700 Subject: [PATCH 33/44] Fixing test bugs [RUN CI][RUN DIALECTS] --- .../trino_transform_bindings.py | 16 ++++++++++++---- tests/test_metadata/bodosql_graphs.json | 2 +- tests/test_pipeline_tpch_custom.py | 4 +--- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/pydough/sqlglot/transform_bindings/trino_transform_bindings.py b/pydough/sqlglot/transform_bindings/trino_transform_bindings.py index 4cd586f18..a72ba36fe 100644 --- a/pydough/sqlglot/transform_bindings/trino_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/trino_transform_bindings.py @@ -160,10 +160,18 @@ def convert_explode( assert explode_spec.delimiter is not None, ( "Delimiter must be provided for string explode." ) - explode_expr = sqlglot_expressions.Split( - this=explode_expr, - expression=sqlglot_expressions.Literal.string(explode_spec.delimiter), - ) + if explode_spec.delimiter == "": + explode_expr = sqlglot_expressions.Anonymous( + this="regexp_extract_all", + expressions=[explode_expr, sqlglot_expressions.Literal.string(".")], + ) + else: + explode_expr = sqlglot_expressions.Split( + this=explode_expr, + expression=sqlglot_expressions.Literal.string( + explode_spec.delimiter + ), + ) explode_op: SQLGlotExpression if val_index is None: explode_op = Unnest( diff --git a/tests/test_metadata/bodosql_graphs.json b/tests/test_metadata/bodosql_graphs.json index 6ef0b19bf..64a56ad76 100644 --- a/tests/test_metadata/bodosql_graphs.json +++ b/tests/test_metadata/bodosql_graphs.json @@ -260,7 +260,7 @@ "always matches": true, "description": "The company that handled the shipment" } - ] + ], "additional definitions": [], "verified pydough analysis": [], "extra semantic info": {} diff --git a/tests/test_pipeline_tpch_custom.py b/tests/test_pipeline_tpch_custom.py index decc9076d..f2c07764a 100644 --- a/tests/test_pipeline_tpch_custom.py +++ b/tests/test_pipeline_tpch_custom.py @@ -7127,9 +7127,7 @@ def test_pipeline_e2e_simple_week( pytest.param( dataframe_collection_bad_5, None, - re.escape( - "Arrays in column 'col1', are not supported for dataframe collections" - ), + re.escape("Array types are not currently supported in dialect SQLITE"), id="dataframe_collection_bad_5", ), pytest.param( From ad51c98971d7104c290deec8d878ed0309f5a32c Mon Sep 17 00:00:00 2001 From: knassre-bodo Date: Thu, 20 Aug 2026 10:47:20 -0700 Subject: [PATCH 34/44] Revisions --- documentation/dsl.md | 1 + pydough/conversion/agg_removal.py | 7 +++++++ pydough/conversion/relational_converter.py | 4 +++- .../database_connectors/database_connector.py | 2 +- .../oracle_transform_bindings.py | 17 ----------------- pydough/unqualified/qualification.py | 3 +++ 6 files changed, 15 insertions(+), 19 deletions(-) diff --git a/documentation/dsl.md b/documentation/dsl.md index 4ccf95edf..882b7575c 100644 --- a/documentation/dsl.md +++ b/documentation/dsl.md @@ -1728,6 +1728,7 @@ infinity value with `DatabaseDiatect.MYSQL` an error will be raised. > [!IMPORTANT] > `ArrayType` is only supported for certain dialects: Trino, Postgres, DuckDB, Databricks. +> Postgres has a limited ability to support rows with an empty array, depending on the type of the column. These sorts of array literals are only supported when the overall column type is an array of booleans, numbers, strings, or datetime values. #### Example 1 diff --git a/pydough/conversion/agg_removal.py b/pydough/conversion/agg_removal.py index b24d43e2f..87a5cf205 100644 --- a/pydough/conversion/agg_removal.py +++ b/pydough/conversion/agg_removal.py @@ -225,6 +225,13 @@ def deduce_explode_uniqueness( """ Helper function to transforms the uniqueness sets after an Explode operation duplicates rows. + + Args: + `unique_terms`: the uniqueness sets of the input to the explode. + `explode`: the explode node. + + Returns: + The uniqueness sets of the output of the explode. """ # Build up the list of column names that imply uniqueness within one of the # expanded row sets (either the index column, or the expanded data, or diff --git a/pydough/conversion/relational_converter.py b/pydough/conversion/relational_converter.py index 16ddbb94f..20377de90 100644 --- a/pydough/conversion/relational_converter.py +++ b/pydough/conversion/relational_converter.py @@ -1408,7 +1408,9 @@ def translate_explode( self, operation: HybridExplode, context: TranslationOutput ) -> TranslationOutput: """ - TODO + Converts an HybridExplode operation into the relational tree for the + EXPLODE operation, which unnests a collection column into multiple rows, + with the exploded values appearing in separate rows. """ exploded_data: RelationalExpression = self.translate_expression( operation.explode_data.shift_back(-1), context diff --git a/pydough/database_connectors/database_connector.py b/pydough/database_connectors/database_connector.py index 0a070f153..620e8956a 100644 --- a/pydough/database_connectors/database_connector.py +++ b/pydough/database_connectors/database_connector.py @@ -76,7 +76,7 @@ def execute_query_df(self, sql: str) -> pd.DataFrame: } ) case DatabaseDialect.MYSQL: - # Snowflake returns JSON data (type 245) as strings, so we + # MySQL returns JSON data (type 245) as strings, so we # need to parse those back into Python types. semi_structured_cols.update( { diff --git a/pydough/sqlglot/transform_bindings/oracle_transform_bindings.py b/pydough/sqlglot/transform_bindings/oracle_transform_bindings.py index 1b9ec57a1..a290bb4bf 100644 --- a/pydough/sqlglot/transform_bindings/oracle_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/oracle_transform_bindings.py @@ -102,23 +102,6 @@ def convert_listof( ) -> SQLGlotExpression: return sqlglot_expressions.Anonymous(this="JSON_ARRAYAGG", expressions=args) - def generate_dataframe_array_expression( - self, items: list[SQLGlotExpression], inner_type: PyDoughType - ) -> SQLGlotExpression: - func: str - match inner_type: - case StringType(): - func = "SYS.ODCIVARCHAR2LIST" - case NumericType(): - func = "SYS.ODCINUMBERLIST" - case DatetimeType(): - func = "SYS.ODCIDATELIST" - case _: - raise ValueError( - f"Cannot support constant array of type {inner_type} in Oracle." - ) - return sqlglot_expressions.Anonymous(this=func, expressions=items) - def convert_sum( self, args: list[SQLGlotExpression], types: list[PyDoughType] ) -> SQLGlotExpression: diff --git a/pydough/unqualified/qualification.py b/pydough/unqualified/qualification.py index 162e7793c..a5907e588 100644 --- a/pydough/unqualified/qualification.py +++ b/pydough/unqualified/qualification.py @@ -1293,6 +1293,9 @@ def qualify_explode( evaluated within. `is_child`: whether the collection is being qualified as a child of a child operator context, such as CALCULATE or PARTITION. + + Returns: + The PyDough QDAG object for the qualified EXPLODE node. """ unqualified_parent: UnqualifiedNode = unqualified._parcel[0] data_raw: UnqualifiedNode = unqualified._parcel[1] From ce0979d6a5431910934ae1c11bff1002edeadff6 Mon Sep 17 00:00:00 2001 From: knassre-bodo Date: Thu, 20 Aug 2026 10:57:11 -0700 Subject: [PATCH 35/44] Adding explode literal array test --- pydough/unqualified/unqualified_node.py | 2 +- tests/test_pipeline_tpch_custom.py | 45 +++++++++++++++++++ tests/test_plan_refsols/explode_22.txt | 4 ++ .../explode_22_databricks.sql | 6 +++ tests/test_sql_refsols/explode_22_duckdb.sql | 10 +++++ .../test_sql_refsols/explode_22_postgres.sql | 6 +++ .../test_sql_refsols/explode_22_snowflake.sql | 6 +++ tests/test_sql_refsols/explode_22_trino.sql | 6 +++ 8 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 tests/test_plan_refsols/explode_22.txt create mode 100644 tests/test_sql_refsols/explode_22_databricks.sql create mode 100644 tests/test_sql_refsols/explode_22_duckdb.sql create mode 100644 tests/test_sql_refsols/explode_22_postgres.sql create mode 100644 tests/test_sql_refsols/explode_22_snowflake.sql create mode 100644 tests/test_sql_refsols/explode_22_trino.sql diff --git a/pydough/unqualified/unqualified_node.py b/pydough/unqualified/unqualified_node.py index 24e0e7b2d..d3f101773 100644 --- a/pydough/unqualified/unqualified_node.py +++ b/pydough/unqualified/unqualified_node.py @@ -898,7 +898,7 @@ def __init__( ExplodeSpec, ] = ( predecessor, - data, + self.coerce_to_unqualified(data), name, explode_spec, ) diff --git a/tests/test_pipeline_tpch_custom.py b/tests/test_pipeline_tpch_custom.py index e4d3d3ccb..1ea9b9a51 100644 --- a/tests/test_pipeline_tpch_custom.py +++ b/tests/test_pipeline_tpch_custom.py @@ -5334,6 +5334,51 @@ ), id="explode_21", ), + pytest.param( + PyDoughPandasTest( + "result = (" + " regions" + " .CALCULATE(name)" + " .EXPLODE(['A', 'E', 'I'], 'letters', value_name='letter', index_name='idx')" + " .CALCULATE(name, letter)" + " .WHERE(CONTAINS(name, letter))" + ")", + "TPCH", + lambda: pd.DataFrame( + { + "name": [ + "AFRICA", + "AFRICA", + "AMERICA", + "AMERICA", + "AMERICA", + "ASIA", + "ASIA", + "EUROPE", + "MIDDLE EAST", + "MIDDLE EAST", + "MIDDLE EAST", + ], + "letter": [ + "A", + "I", + "A", + "E", + "I", + "A", + "I", + "E", + "A", + "E", + "I", + ], + } + ), + "explode_22", + skipped_dialects={"ANSI", "SQLITE", "ORACLE", "MYSQL"}, + ), + id="explode_22", + ), pytest.param( PyDoughPandasTest( simple_range_1, diff --git a/tests/test_plan_refsols/explode_22.txt b/tests/test_plan_refsols/explode_22.txt new file mode 100644 index 000000000..edb833423 --- /dev/null +++ b/tests/test_plan_refsols/explode_22.txt @@ -0,0 +1,4 @@ +ROOT(columns=[('name', name), ('letter', letter)], orderings=[]) + FILTER(condition=CONTAINS(name, letter), columns={'letter': letter, 'name': name}) + EXPLODE(['A', 'E', 'I']:array[unknown], value_name='letter', index_name='idx', version='array', filtering=True, is_distinct=False, columns={'idx': idx, 'letter': letter, 'name': name}) + SCAN(table=tpch.REGION, columns={'name': r_name}) diff --git a/tests/test_sql_refsols/explode_22_databricks.sql b/tests/test_sql_refsols/explode_22_databricks.sql new file mode 100644 index 000000000..7e4df8968 --- /dev/null +++ b/tests/test_sql_refsols/explode_22_databricks.sql @@ -0,0 +1,6 @@ +SELECT + region.r_name AS name, + _s0.val AS letter +FROM tpch.region AS region, LATERAL POSEXPLODE(ARRAY('A', 'E', 'I')) AS _s0(idx, val) +WHERE + CONTAINS(region.r_name, _s0.val) diff --git a/tests/test_sql_refsols/explode_22_duckdb.sql b/tests/test_sql_refsols/explode_22_duckdb.sql new file mode 100644 index 000000000..95df81669 --- /dev/null +++ b/tests/test_sql_refsols/explode_22_duckdb.sql @@ -0,0 +1,10 @@ +SELECT + region.r_name AS name, + _s0.val AS letter +FROM tpch.region AS region, LATERAL ( + SELECT + UNNEST(['A', 'E', 'I']) AS _col_0, + GENERATE_SUBSCRIPTS(['A', 'E', 'I'], 1) - 1 AS _col_1 +) AS _s0(val, idx) +WHERE + region.r_name LIKE CONCAT('%', _s0.val, '%') diff --git a/tests/test_sql_refsols/explode_22_postgres.sql b/tests/test_sql_refsols/explode_22_postgres.sql new file mode 100644 index 000000000..8fa89512e --- /dev/null +++ b/tests/test_sql_refsols/explode_22_postgres.sql @@ -0,0 +1,6 @@ +SELECT + region.r_name AS name, + _s0.val AS letter +FROM tpch.region AS region, LATERAL UNNEST(ARRAY['A', 'E', 'I']) WITH ORDINALITY AS _s0(val, idx) +WHERE + region.r_name LIKE CONCAT('%', _s0.val, '%') diff --git a/tests/test_sql_refsols/explode_22_snowflake.sql b/tests/test_sql_refsols/explode_22_snowflake.sql new file mode 100644 index 000000000..c80d64bf5 --- /dev/null +++ b/tests/test_sql_refsols/explode_22_snowflake.sql @@ -0,0 +1,6 @@ +SELECT + region.r_name AS name, + _s0.value AS letter +FROM tpch.region AS region, LATERAL FLATTEN(['A', 'E', 'I']) AS _s0(seq, key, path, index, value, this) +WHERE + CONTAINS(region.r_name, _s0.value) diff --git a/tests/test_sql_refsols/explode_22_trino.sql b/tests/test_sql_refsols/explode_22_trino.sql new file mode 100644 index 000000000..4669c1a68 --- /dev/null +++ b/tests/test_sql_refsols/explode_22_trino.sql @@ -0,0 +1,6 @@ +SELECT + region.r_name AS name, + _s0.val AS letter +FROM tpch.region AS region, UNNEST(ARRAY['A', 'E', 'I']) WITH ORDINALITY AS _s0(val, idx) +WHERE + region.r_name LIKE CONCAT('%', _s0.val, '%') From 2d606475d91c8e62c32cbb551c31c1fe70138282 Mon Sep 17 00:00:00 2001 From: knassre-bodo Date: Thu, 20 Aug 2026 11:06:17 -0700 Subject: [PATCH 36/44] Added exploding string literal test --- .../databricks_transform_bindings.py | 30 ++++++++++++++----- tests/test_pipeline_tpch_custom.py | 19 ++++++++++++ tests/test_plan_refsols/explode_23.txt | 3 ++ .../explode_23_databricks.sql | 7 +++++ tests/test_sql_refsols/explode_23_duckdb.sql | 10 +++++++ .../test_sql_refsols/explode_23_postgres.sql | 6 ++++ .../test_sql_refsols/explode_23_snowflake.sql | 6 ++++ tests/test_sql_refsols/explode_23_trino.sql | 6 ++++ 8 files changed, 79 insertions(+), 8 deletions(-) create mode 100644 tests/test_plan_refsols/explode_23.txt create mode 100644 tests/test_sql_refsols/explode_23_databricks.sql create mode 100644 tests/test_sql_refsols/explode_23_duckdb.sql create mode 100644 tests/test_sql_refsols/explode_23_postgres.sql create mode 100644 tests/test_sql_refsols/explode_23_snowflake.sql create mode 100644 tests/test_sql_refsols/explode_23_trino.sql diff --git a/pydough/sqlglot/transform_bindings/databricks_transform_bindings.py b/pydough/sqlglot/transform_bindings/databricks_transform_bindings.py index cc3b527fa..90b4a53a2 100644 --- a/pydough/sqlglot/transform_bindings/databricks_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/databricks_transform_bindings.py @@ -113,20 +113,22 @@ def convert_explode( subquery_alias: str, ) -> SQLGlotExpression: column_exprs: list[SQLGlotExpression] = [*exprs] + val_expr: SQLGlotExpression = sqlglot_expressions.Column( + this=sqlglot_expressions.Identifier(this="val"), + table=sqlglot_expressions.Identifier(this=lateral_alias), + ) + idx_expr: SQLGlotExpression = sqlglot_expressions.Column( + this=sqlglot_expressions.Identifier(this="idx"), + table=sqlglot_expressions.Identifier(this=lateral_alias), + ) if val_index is not None: column_exprs[val_index] = sqlglot_expressions.Alias( - this=sqlglot_expressions.Column( - this=sqlglot_expressions.Identifier(this="val"), - table=sqlglot_expressions.Identifier(this=lateral_alias), - ), + this=val_expr, alias=sqlglot_expressions.Identifier(this=explode_spec.value_name), ) if idx_index is not None and explode_spec.index_name is not None: column_exprs[idx_index] = sqlglot_expressions.Alias( - this=sqlglot_expressions.Column( - this=sqlglot_expressions.Identifier(this="idx"), - table=sqlglot_expressions.Identifier(this=lateral_alias), - ), + this=idx_expr, alias=sqlglot_expressions.Identifier(this=explode_spec.index_name), ) @@ -166,6 +168,18 @@ def convert_explode( ) ) + if explode_spec.version == "string" and explode_spec.delimiter == "": + # Databricks' SPLIT() returns an array with a single empty string + # when the input is an empty string, but PyDough's EXPLODE() expects + # no rows to be returned in that case. Filter out the empty string + # from the exploded results. + result = result.where( + sqlglot_expressions.NEQ( + this=val_expr, + expression=sqlglot_expressions.Literal.string(""), + ) + ) + return result def generate_dataframe_item_dialect_expression( diff --git a/tests/test_pipeline_tpch_custom.py b/tests/test_pipeline_tpch_custom.py index 1ea9b9a51..01a4b90fc 100644 --- a/tests/test_pipeline_tpch_custom.py +++ b/tests/test_pipeline_tpch_custom.py @@ -5379,6 +5379,25 @@ ), id="explode_22", ), + pytest.param( + PyDoughPandasTest( + "result = (" + " TPCH" + " .EXPLODE('ALPHABET', 'letters', value_name='letter', index_name='idx', version='string', delimiter='')" + " .CALCULATE(idx, letter)" + ")", + "TPCH", + lambda: pd.DataFrame( + { + "idx": list(range(8)), + "letter": list("ALPHABET"), + } + ), + "explode_23", + skipped_dialects={"ANSI", "SQLITE", "ORACLE", "MYSQL"}, + ), + id="explode_23", + ), pytest.param( PyDoughPandasTest( simple_range_1, diff --git a/tests/test_plan_refsols/explode_23.txt b/tests/test_plan_refsols/explode_23.txt new file mode 100644 index 000000000..4f3f9276e --- /dev/null +++ b/tests/test_plan_refsols/explode_23.txt @@ -0,0 +1,3 @@ +ROOT(columns=[('idx', idx), ('letter', letter)], orderings=[]) + EXPLODE('ALPHABET':string, value_name='letter', index_name='idx', version='string', delimiter='', filtering=True, is_distinct=False, columns={'idx': idx, 'letter': letter}) + EMPTYSINGLETON() diff --git a/tests/test_sql_refsols/explode_23_databricks.sql b/tests/test_sql_refsols/explode_23_databricks.sql new file mode 100644 index 000000000..d0ba95419 --- /dev/null +++ b/tests/test_sql_refsols/explode_23_databricks.sql @@ -0,0 +1,7 @@ +SELECT + _s0.idx, + _s0.val AS letter +FROM VALUES + (NULL) AS _q_0(_col_0), LATERAL POSEXPLODE(SPLIT('ALPHABET', '\\Q\\E')) AS _s0(idx, val) +WHERE + _s0.val <> '' diff --git a/tests/test_sql_refsols/explode_23_duckdb.sql b/tests/test_sql_refsols/explode_23_duckdb.sql new file mode 100644 index 000000000..04be5e29a --- /dev/null +++ b/tests/test_sql_refsols/explode_23_duckdb.sql @@ -0,0 +1,10 @@ +SELECT + _s0.idx, + _s0.val AS letter +FROM (VALUES + (NULL)) AS _q_0(_col_0) +CROSS JOIN LATERAL ( + SELECT + UNNEST(REGEXP_SPLIT_TO_ARRAY('ALPHABET', '')) AS _col_0, + GENERATE_SUBSCRIPTS(REGEXP_SPLIT_TO_ARRAY('ALPHABET', ''), 1) - 1 AS _col_1 +) AS _s0(val, idx) diff --git a/tests/test_sql_refsols/explode_23_postgres.sql b/tests/test_sql_refsols/explode_23_postgres.sql new file mode 100644 index 000000000..662e0a709 --- /dev/null +++ b/tests/test_sql_refsols/explode_23_postgres.sql @@ -0,0 +1,6 @@ +SELECT + _s0.idx - 1 AS idx, + _s0.val AS letter +FROM (VALUES + (NULL)) AS _q_0(_col_0) +CROSS JOIN LATERAL UNNEST(REGEXP_SPLIT_TO_ARRAY('ALPHABET', '')) WITH ORDINALITY AS _s0(val, idx) diff --git a/tests/test_sql_refsols/explode_23_snowflake.sql b/tests/test_sql_refsols/explode_23_snowflake.sql new file mode 100644 index 000000000..53b842e42 --- /dev/null +++ b/tests/test_sql_refsols/explode_23_snowflake.sql @@ -0,0 +1,6 @@ +SELECT + _s0.index AS idx, + _s0.value AS letter +FROM (VALUES + (NULL)) AS _q_0(_col_0) +CROSS JOIN LATERAL FLATTEN(REGEXP_EXTRACT_ALL('ALPHABET', '.{1}')) AS _s0(seq, key, path, index, value, this) diff --git a/tests/test_sql_refsols/explode_23_trino.sql b/tests/test_sql_refsols/explode_23_trino.sql new file mode 100644 index 000000000..35f06a6cc --- /dev/null +++ b/tests/test_sql_refsols/explode_23_trino.sql @@ -0,0 +1,6 @@ +SELECT + _s0.idx - 1 AS idx, + _s0.val AS letter +FROM (VALUES + (NULL)) AS _q_0(_col_0) +CROSS JOIN UNNEST(REGEXP_EXTRACT_ALL('ALPHABET', '.')) WITH ORDINALITY AS _s0(val, idx) From 4ff27364845e5d7b3fa179ec7fd791d91a8c20e3 Mon Sep 17 00:00:00 2001 From: knassre-bodo Date: Fri, 21 Aug 2026 09:56:42 -0700 Subject: [PATCH 37/44] Revisions, testing fixes [RUN CI][RUN DIALECTS] --- documentation/dsl.md | 28 +++++++++++++++++++++ tests/test_sql_refsols/explode_20_trino.sql | 2 +- tests/test_sql_refsols/explode_21_trino.sql | 2 +- tests/testing_utilities.py | 14 +++++++++-- 4 files changed, 42 insertions(+), 4 deletions(-) diff --git a/documentation/dsl.md b/documentation/dsl.md index 882b7575c..6c278a589 100644 --- a/documentation/dsl.md +++ b/documentation/dsl.md @@ -1593,6 +1593,34 @@ People.EXPLODE(LOWER(first_name), "characters", value_name='char', index_name="i .CALCULATE(char, n_uses=COUNT(characters)) ``` +**Data Example**: + +Suppose we have the following collection of data `thesaurus` containing words and some of their synonyms in an array: +| word | synonyms | +|---------|--------------------------------| +| 'wise' | ['sage', 'insightful', 'keen'] | +| 'old' | ['elderly', 'ancient'] | +| 'my' | [] | +| 'large' | ['big'] | + +Now suppose the following PyDough code is used to transform `thesaurus` using the `EXPLODE` operator (note: `filtering=True` because one of the rows is an empty array): +```py +%%pydough +thesaurus.CALCULATE(word) + .EXPLODE(synonyms, "words", value_name='synonym', index_name='syn_idx', filtering=True) + .CALCULATE(word, syn_idx, synonym) +``` + +The result would be the following table: +| name | syn_idx | synonym | +|---------|---------|--------------| +| 'wise' | 0 | 'sage' | +| 'wise' | 1 | 'insightful' | +| 'wise' | 2 | 'keen' | +| 'old' | 0 | 'elderly | +| 'old' | 1 | 'ancient | +| 'large' | 0 | 'big' | + **Bad Example #1**: Missing the `name`. ```py diff --git a/tests/test_sql_refsols/explode_20_trino.sql b/tests/test_sql_refsols/explode_20_trino.sql index 8b6dbad5f..d1cbb6c9d 100644 --- a/tests/test_sql_refsols/explode_20_trino.sql +++ b/tests/test_sql_refsols/explode_20_trino.sql @@ -9,7 +9,7 @@ WITH _s1 AS ( SELECT _s0.val AS char, COUNT(*) AS n -FROM _s1 AS _s1, UNNEST(SPLIT(_s1.comment, '')) WITH ORDINALITY AS _s0(val, idx) +FROM _s1 AS _s1, UNNEST(REGEXP_EXTRACT_ALL(_s1.comment, '.')) WITH ORDINALITY AS _s0(val, idx) WHERE _s0.val <> '' GROUP BY diff --git a/tests/test_sql_refsols/explode_21_trino.sql b/tests/test_sql_refsols/explode_21_trino.sql index e7da4f3ad..0575530e7 100644 --- a/tests/test_sql_refsols/explode_21_trino.sql +++ b/tests/test_sql_refsols/explode_21_trino.sql @@ -18,7 +18,7 @@ WITH _s1 AS ( FROM _s1 AS _s1 CROSS JOIN UNNEST(SPLIT(_s1.comment, '.')) WITH ORDINALITY AS _s0(val, idx) CROSS JOIN UNNEST(SPLIT(_s0.val, ',')) WITH ORDINALITY AS _s2(val, idx) - CROSS JOIN UNNEST(SPLIT(_s2.val, ' ')) WITH ORDINALITY AS _s4(val, idx), UNNEST(SPLIT(_s4.val, '')) WITH ORDINALITY AS _s6(val, idx) + CROSS JOIN UNNEST(SPLIT(_s2.val, ' ')) WITH ORDINALITY AS _s4(val, idx), UNNEST(REGEXP_EXTRACT_ALL(_s4.val, '.')) WITH ORDINALITY AS _s6(val, idx) WHERE _s6.val <> '' ) diff --git a/tests/testing_utilities.py b/tests/testing_utilities.py index 957ac8b46..aa5263537 100644 --- a/tests/testing_utilities.py +++ b/tests/testing_utilities.py @@ -1580,10 +1580,20 @@ def run_e2e_test( if self.ignore_array_order and len(result) > 1 and len(refsol) > 1: for col in result.columns: result[col] = result[col].apply( - lambda x: sorted(x) if isinstance(x, (list, np.ndarray)) else x + lambda x: ( + sorted(x) + if isinstance(x, (list, np.ndarray)) + or (hasattr(x, "__iter__") and not isinstance(x, str)) + else x + ) ) refsol[col] = refsol[col].apply( - lambda x: sorted(x) if isinstance(x, (list, np.ndarray)) else x + lambda x: ( + sorted(x) + if isinstance(x, (list, np.ndarray)) + or (hasattr(x, "__iter__") and not isinstance(x, str)) + else x + ) ) # Perform the comparison between the result and the reference solution From 2adbb90e91201640d12e46ca717c61c6ca77b272 Mon Sep 17 00:00:00 2001 From: knassre-bodo Date: Fri, 21 Aug 2026 10:28:06 -0700 Subject: [PATCH 38/44] Fixed bugs, added docs, test run to epxpose trino issue [RUN TRINO] --- .../databricks_transform_bindings.py | 26 ++++++++++++++ .../duckdb_transform_bindings.py | 34 +++++++++++++++++++ .../postgres_transform_bindings.py | 25 ++++++++++++++ .../sf_transform_bindings.py | 25 ++++++++++++++ .../trino_transform_bindings.py | 25 ++++++++++++++ tests/test_pipeline_tpch.py | 5 --- tests/test_pipeline_tpch_custom.py | 12 +------ tests/testing_utilities.py | 14 ++++++-- 8 files changed, 148 insertions(+), 18 deletions(-) diff --git a/pydough/sqlglot/transform_bindings/databricks_transform_bindings.py b/pydough/sqlglot/transform_bindings/databricks_transform_bindings.py index 90b4a53a2..da2d3fb6c 100644 --- a/pydough/sqlglot/transform_bindings/databricks_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/databricks_transform_bindings.py @@ -112,6 +112,32 @@ def convert_explode( lateral_alias: str, subquery_alias: str, ) -> SQLGlotExpression: + """ + What the final SQL will look like for array explosion: + + ``` + SELECT ..., L.val AS value, L.idx AS index + FROM (...) AS S + CROSS JOIN POSEXPLODE(explode_expr) AS L(val, idx) + ``` + + What the final SQL will look like for string explosion (regular): + + ``` + SELECT ..., L.val AS value, L.idx AS index + FROM (...) AS S, + CROSS JOIN POSEXPLODE(SPLIT(explode_expr, delimiter)) AS L(val, idx) + ``` + + What the final SQL will look like for string explosion (no delimiter): + + ``` + SELECT ..., L.val AS value, L.idx AS index + FROM (...) AS S, + CROSS JOIN POSEXPLODE(SPLIT(explode_expr, '\\Q.\\E')) AS L(val, idx) + WHERE L.val != '' + ``` + """ column_exprs: list[SQLGlotExpression] = [*exprs] val_expr: SQLGlotExpression = sqlglot_expressions.Column( this=sqlglot_expressions.Identifier(this="val"), diff --git a/pydough/sqlglot/transform_bindings/duckdb_transform_bindings.py b/pydough/sqlglot/transform_bindings/duckdb_transform_bindings.py index eeed84392..811f6118c 100644 --- a/pydough/sqlglot/transform_bindings/duckdb_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/duckdb_transform_bindings.py @@ -149,6 +149,40 @@ def convert_explode( lateral_alias: str, subquery_alias: str, ) -> SQLGlotExpression: + """ + What the final SQL will look like for array explosion: + + ``` + SELECT ..., L.val AS value, L.idx AS index + FROM (...) AS S, + LATERAL ( + UNNEST(explode_expr) as val, + GENERATE_SUBSCRIPTS(explode_expr, 1) as idx + ) AS L + ``` + + What the final SQL will look like for string explosion (regular): + + ``` + SELECT ..., L.val AS value, L.idx AS index + FROM (...) AS S, + LATERAL ( + UNNEST(SPLIT(explode_expr, delimiter)) as val, + GENERATE_SUBSCRIPTS(SPLIT(explode_expr, delimiter), 1) as idx + ) AS L + ``` + + What the final SQL will look like for string explosion (no delimiter): + + ``` + SELECT ..., L.val AS value, L.idx AS index + FROM (...) AS S, + LATERAL ( + UNNEST(REGEXP_SPLIT_TO_ARRAY(explode_expr, '')) as val, + GENERATE_SUBSCRIPTS(REGEXP_SPLIT_TO_ARRAY(explode_expr, ''), 1) as idx + ) AS L + ``` + """ column_exprs: list[SQLGlotExpression] = [*exprs] if val_index is not None: column_exprs[val_index] = sqlglot_expressions.Alias( diff --git a/pydough/sqlglot/transform_bindings/postgres_transform_bindings.py b/pydough/sqlglot/transform_bindings/postgres_transform_bindings.py index 09396937c..8109e414b 100644 --- a/pydough/sqlglot/transform_bindings/postgres_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/postgres_transform_bindings.py @@ -87,6 +87,31 @@ def convert_explode( lateral_alias: str, subquery_alias: str, ) -> SQLGlotExpression: + """ + What the final SQL will look like for array explosion: + + ``` + SELECT ..., L.val AS value, L.idx - 1 AS index + FROM (...) AS S + CROSS JOIN LATERAL UNNEST(explode_expr) WITH ORDINALITY AS L(val, idx) + ``` + + What the final SQL will look like for string explosion (regular): + + ``` + SELECT ..., L.val AS value, L.idx - 1 AS index + FROM (...) AS S, + CROSS JOIN LATERAL UNNEST(STRING_TO_ARRAY(explode_expr, delimiter)) WITH ORDINALITY AS L(val, idx) + ``` + + What the final SQL will look like for string explosion (no delimiter): + + ``` + SELECT ..., L.val AS value, L.idx - 1 AS index + FROM (...) AS S, + CROSS JOIN LATERAL UNNEST(REGEXP_SPLIT_TO_ARRAY(explode_expr, '')) WITH ORDINALITY AS L(val, idx) + ``` + """ column_exprs: list[SQLGlotExpression] = [*exprs] if val_index is not None: column_exprs[val_index] = sqlglot_expressions.Alias( diff --git a/pydough/sqlglot/transform_bindings/sf_transform_bindings.py b/pydough/sqlglot/transform_bindings/sf_transform_bindings.py index e2be6131c..a18f41ac9 100644 --- a/pydough/sqlglot/transform_bindings/sf_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/sf_transform_bindings.py @@ -258,6 +258,31 @@ def convert_explode( lateral_alias: str, subquery_alias: str, ) -> SQLGlotExpression: + """ + What the final SQL will look like for array explosion: + + ``` + SELECT ..., L.value AS value, L.index AS idx + FROM (...) AS S, + LATERAL FLATTEN(explode_expr) AS L(seq, key, path, index, value, this) + ``` + + What the final SQL will look like for string explosion (regular): + + ``` + SELECT ..., L.value AS value, L.index - 1 AS idx + FROM (...) AS S, + LATERAL SPLIT_TO_TABLE(explode_expr, delimiter) AS L + ``` + + What the final SQL will look like for string explosion (no delimiter): + + ``` + SELECT ..., L.value AS value, L.index AS idx + FROM (...) AS S, + LATERAL FLATTEN(REGEXP_SUBSTR_ALL(explode_expr, '.{1}')) AS L + ``` + """ # Rewrite all input expressions to point to the subquery column_exprs: list[SQLGlotExpression] = [] for expr in exprs: diff --git a/pydough/sqlglot/transform_bindings/trino_transform_bindings.py b/pydough/sqlglot/transform_bindings/trino_transform_bindings.py index e6d5eb519..77ee089d5 100644 --- a/pydough/sqlglot/transform_bindings/trino_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/trino_transform_bindings.py @@ -135,6 +135,31 @@ def convert_explode( lateral_alias: str, subquery_alias: str, ) -> SQLGlotExpression: + """ + What the final SQL will look like for array explosion: + + ``` + SELECT ..., L.val AS value, L.idx AS index + FROM (...) AS S, + LATERAL UNNEST(explode_expr) WITH ORDINALITY AS L(val, idx) + ``` + + What the final SQL will look like for string explosion (regular): + + ``` + SELECT ..., L.val AS value, L.idx AS index + FROM (...) AS S, + LATERAL UNNEST(SPLIT(explode_expr, delimiter)) WITH ORDINALITY AS L(val, idx) + ``` + + What the final SQL will look like for string explosion (no delimiter): + + ``` + SELECT ..., L.val AS value, L.idx AS index + FROM (...) AS S, + LATERAL UNNEST(REGEXP_EXTRACT_ALL(explode_expr, '.')) WITH ORDINALITY AS L(val, idx) + ``` + """ column_exprs: list[SQLGlotExpression] = [*exprs] if val_index is not None: column_exprs[val_index] = sqlglot_expressions.Alias( diff --git a/tests/test_pipeline_tpch.py b/tests/test_pipeline_tpch.py index 092c3f05c..18fc0a8b8 100644 --- a/tests/test_pipeline_tpch.py +++ b/tests/test_pipeline_tpch.py @@ -44,11 +44,6 @@ def test_pipeline_until_sql_tpch( """ Same as test_pipeline_until_relational_tpch, but for the generated SQL text. """ - if ( - tpch_pipeline_test_data.test_name == "dataframe_collection_inf" - and empty_context_database.dialect == DatabaseDialect.MYSQL - ): - pytest.skip("Skipping test as MySQL does not support Infinity values.") file_path: str = get_sql_test_filename( tpch_pipeline_test_data.test_name, empty_context_database.dialect diff --git a/tests/test_pipeline_tpch_custom.py b/tests/test_pipeline_tpch_custom.py index 01a4b90fc..646da5f10 100644 --- a/tests/test_pipeline_tpch_custom.py +++ b/tests/test_pipeline_tpch_custom.py @@ -5899,6 +5899,7 @@ } ), "dataframe_collection_inf", + skipped_dialects={"MYSQL"}, ), id="dataframe_collection_inf", ), @@ -6566,11 +6567,6 @@ def test_pipeline_until_sql_tpch_custom( """ Same as test_pipeline_until_relational_tpch, but for the generated SQL text. """ - if ( - tpch_custom_pipeline_test_data.test_name == "dataframe_collection_inf" - and empty_context_database.dialect == DatabaseDialect.MYSQL - ): - pytest.skip("Skipping test as MySQL does not support Infinity values.") tpch_custom_pipeline_test_data = tpch_custom_test_data_dialect_replacements( empty_context_database.dialect, tpch_custom_pipeline_test_data @@ -6600,12 +6596,6 @@ def test_pipeline_e2e_tpch_custom( if db_context.dialect == DatabaseDialect.BODOSQL: pytest.skip("Skipping tpch custom test for BodoSQL.") - if ( - db_context.dialect == DatabaseDialect.MYSQL - and tpch_custom_pipeline_test_data.test_name == "dataframe_collection_inf" - ): - pytest.skip("Skipping test as MySQL does not support Infinity values.") - tpch_custom_pipeline_test_data = tpch_custom_test_data_dialect_replacements( db_context.dialect, tpch_custom_pipeline_test_data ) diff --git a/tests/testing_utilities.py b/tests/testing_utilities.py index aa5263537..6ffb0a604 100644 --- a/tests/testing_utilities.py +++ b/tests/testing_utilities.py @@ -1579,6 +1579,13 @@ def run_e2e_test( # within arrays caused by how different dialects group array values. if self.ignore_array_order and len(result) > 1 and len(refsol) > 1: for col in result.columns: + print( + col, + type(result[col][0]), + result[col][0], + type(refsol[col][0]), + refsol[col][0], + ) result[col] = result[col].apply( lambda x: ( sorted(x) @@ -1800,8 +1807,11 @@ def sanitize_string(val): ) # float vs None. Convert to nullable floats - if all(isinstance(elem, (float, NoneType)) for elem in column_a) and all( - isinstance(elem, (float, NoneType)) for elem in column_b + if ( + all(isinstance(elem, (float, NoneType)) for elem in column_a) + and all(isinstance(elem, (float, NoneType)) for elem in column_b) + and len(column_a) > 0 + and len(column_b) > 0 ): return column_a.astype("Float64"), column_b.astype("Float64") From c8f706bccfe48d273538a8deaac47042fd276e93 Mon Sep 17 00:00:00 2001 From: knassre-bodo Date: Fri, 21 Aug 2026 15:10:04 -0700 Subject: [PATCH 39/44] Fixing trino bug, adding BodoSQL test [RUN CI][RUN DIALECTS] --- tests/test_pipeline_bodosql.py | 16 ++++++++++++++++ tests/test_pipeline_tpch_custom.py | 1 + tests/test_plan_refsols/color_q18.txt | 5 +++++ tests/test_sql_refsols/color_q18_bodosql.sql | 13 +++++++++++++ tests/testing_utilities.py | 7 ------- 5 files changed, 35 insertions(+), 7 deletions(-) create mode 100644 tests/test_plan_refsols/color_q18.txt create mode 100644 tests/test_sql_refsols/color_q18_bodosql.sql diff --git a/tests/test_pipeline_bodosql.py b/tests/test_pipeline_bodosql.py index cff842af7..092099831 100644 --- a/tests/test_pipeline_bodosql.py +++ b/tests/test_pipeline_bodosql.py @@ -1478,6 +1478,22 @@ def impl(name: str) -> GraphMetadata: ), id="color_q16", ), + pytest.param( + # List the most frequent words that appear within color names + PyDoughPandasTest( + "result = (" + " colors" + " .EXPLODE(key, 'words', value_name='word', index_name='idx', version='string', delimiter='_')" + " .PARTITION(name='word_groups', by=word)" + " .BEST(by=COUNT(words).DESC(), allow_ties=True)" + " .CALCULATE(word)" + ")", + "COLORSHOP", + lambda: pd.DataFrame({"word": ["blue"]}), + "color_q18", + ), + id="color_q18", + ), pytest.param( # List all colors whose name ends with `yz`. PyDoughPandasTest( diff --git a/tests/test_pipeline_tpch_custom.py b/tests/test_pipeline_tpch_custom.py index 646da5f10..f3fe9c687 100644 --- a/tests/test_pipeline_tpch_custom.py +++ b/tests/test_pipeline_tpch_custom.py @@ -4312,6 +4312,7 @@ ), "explode_01", order_sensitive=True, + ignore_array_order=True, skipped_dialects={"ANSI", "SQLITE", "ORACLE", "MYSQL"}, ), id="explode_01", diff --git a/tests/test_plan_refsols/color_q18.txt b/tests/test_plan_refsols/color_q18.txt new file mode 100644 index 000000000..5770cb210 --- /dev/null +++ b/tests/test_plan_refsols/color_q18.txt @@ -0,0 +1,5 @@ +ROOT(columns=[('word', word)], orderings=[]) + FILTER(condition=1:numeric == RANKING(args=[], partition=[], order=[(n_rows):desc_first], allow_ties=True), columns={'word': word}) + AGGREGATE(keys={'word': word}, aggregations={'n_rows': COUNT()}) + EXPLODE(key, value_name='word', index_name='idx', version='string', delimiter='_', filtering=True, is_distinct=False, columns={'idx': idx, 'word': word}) + SCAN(table=CLRS, columns={'key': identname}) diff --git a/tests/test_sql_refsols/color_q18_bodosql.sql b/tests/test_sql_refsols/color_q18_bodosql.sql new file mode 100644 index 000000000..6c38d628c --- /dev/null +++ b/tests/test_sql_refsols/color_q18_bodosql.sql @@ -0,0 +1,13 @@ +WITH _t0 AS ( + SELECT + _s0.value AS word + FROM clrs AS clrs + CROSS JOIN LATERAL SPLIT_TO_TABLE(clrs.identname, '_') AS _s0 + GROUP BY + 1 + QUALIFY + RANK() OVER (ORDER BY COUNT(*) DESC) = 1 +) +SELECT + word +FROM _t0 diff --git a/tests/testing_utilities.py b/tests/testing_utilities.py index 6ffb0a604..b795eb5be 100644 --- a/tests/testing_utilities.py +++ b/tests/testing_utilities.py @@ -1579,13 +1579,6 @@ def run_e2e_test( # within arrays caused by how different dialects group array values. if self.ignore_array_order and len(result) > 1 and len(refsol) > 1: for col in result.columns: - print( - col, - type(result[col][0]), - result[col][0], - type(refsol[col][0]), - refsol[col][0], - ) result[col] = result[col].apply( lambda x: ( sorted(x) From f6b2431ff3d99d115efd330607f2ed8a8d7510d8 Mon Sep 17 00:00:00 2001 From: knassre-bodo Date: Fri, 28 Aug 2026 10:04:30 -0700 Subject: [PATCH 40/44] Explode test fix [RUN CI][RUN DIALECTS] --- documentation/functions.md | 17 +++++++-- tests/test_pipeline_tpch_custom.py | 35 ++++++++++++++----- tests/test_plan_refsols/explode_01.txt | 2 +- tests/test_sql_refsols/color_q16_bodosql.sql | 4 +-- .../explode_01_databricks.sql | 2 +- tests/test_sql_refsols/explode_01_duckdb.sql | 2 +- .../test_sql_refsols/explode_01_postgres.sql | 2 +- .../test_sql_refsols/explode_01_snowflake.sql | 2 +- tests/test_sql_refsols/explode_01_trino.sql | 2 +- 9 files changed, 49 insertions(+), 19 deletions(-) diff --git a/documentation/functions.md b/documentation/functions.md index 58ffd8bd2..c5ad51ddc 100644 --- a/documentation/functions.md +++ b/documentation/functions.md @@ -1103,9 +1103,20 @@ The `LISTOF` function collects a set of values into an array. # For each region, list the names of all nations inside that region Regions.CALCULATE(region_name=name, nation_names=LISTOF(nations.name)) -# For each customer, list the total quantities of all purchases made by that customer. -Customers.CALCULATE(customer_name=name, quantities=LISTOF(orders.lines.quantity)) -``` +# For each customer, list the three largest quantities purchased made by that +# customer. +selected_lines = orders.lines.best(by=quantity.DESC(), per='Customers', n_best=3) +Customers.CALCULATE(customer_name=name, quantities=LISTOF(selected_lines.quantity)) +``` + +The first of these examples would produce the following output: +| region_name | nation_names | +|---------------|---------------------------------------------------------------| +| "AFRICA" | ["ALGERIA", "ETHIOPIA", "KENYA", "MOROCCO", "MOZAMBIQUE"] | +| "AMERICA" | ["ARGENTINA", "BRAZIL", "CANADA", "PERU", "UNITED STATES"] | +| "ASIA" | ["INDIA", "INDONESIA", "JAPAN", "CHINA", "VIETNAM"] | +| "EUROPE" | ["FRANCE", "GERMANY", "ROMANIA", "RUSSIA", "UNITED KINGDOM"] | +| "MIDDLE EAST" | ["EGYPT", "IRAN", "IRAQ", "JORDAN", "SAUDI ARABIA"] | diff --git a/tests/test_pipeline_tpch_custom.py b/tests/test_pipeline_tpch_custom.py index f3fe9c687..85213335d 100644 --- a/tests/test_pipeline_tpch_custom.py +++ b/tests/test_pipeline_tpch_custom.py @@ -4244,7 +4244,7 @@ " array_data" " .EXPLODE(nation_names, 'exploded_nations', index_name='nation_idx', value_name='nation_name', version='array', filtering=False, is_distinct=True)" " .CALCULATE(region_name, nation_names, nation_idx, nation_name)" - " .ORDER_BY(region_name, nation_idx)" + " .ORDER_BY(region_name, nation_name)" ")", "TPCH", lambda: pd.DataFrame( @@ -4280,7 +4280,6 @@ * 5 + [["EGYPT", "IRAN", "IRAQ", "JORDAN", "SAUDI ARABIA"]] * 5 ), - "nation_idx": list(range(5)) * 5, "nation_name": [ "ALGERIA", "ETHIOPIA", @@ -7320,6 +7319,7 @@ def test_pipeline_e2e_errors( {"name": ["CHINA", "INDIA", "INDONESIA", "JAPAN", "VIETNAM"]} ), "to_table_test_1", + skipped_dialects={"BODOSQL"}, ), id="to_table_test_1", ), @@ -7334,6 +7334,7 @@ def test_pipeline_e2e_errors( {"name": ["CHINA", "INDIA", "INDONESIA", "VIETNAM"]} ), "to_table_test_2", + skipped_dialects={"BODOSQL"}, ), id="to_table_test_2", ), @@ -7351,6 +7352,7 @@ def test_pipeline_e2e_errors( } ), "to_table_test_3", + skipped_dialects={"BODOSQL"}, ), id="to_table_test_3", ), @@ -7376,6 +7378,7 @@ def test_pipeline_e2e_errors( } ), "to_table_test_4", + skipped_dialects={"BODOSQL"}, ), id="to_table_test_4", ), @@ -7399,6 +7402,7 @@ def test_pipeline_e2e_errors( } ), "to_table_test_5", + skipped_dialects={"BODOSQL"}, ), id="to_table_test_5", ), @@ -7439,6 +7443,7 @@ def test_pipeline_e2e_errors( } ), "to_table_test_6", + skipped_dialects={"BODOSQL"}, ), id="to_table_test_6", ), @@ -7458,6 +7463,7 @@ def test_pipeline_e2e_errors( } ), "to_table_test_7", + skipped_dialects={"BODOSQL"}, ), id="to_table_test_7", ), @@ -7478,6 +7484,7 @@ def test_pipeline_e2e_errors( } ), "to_table_test_8", + skipped_dialects={"BODOSQL"}, ), id="to_table_test_8", ), @@ -7503,6 +7510,7 @@ def test_pipeline_e2e_errors( } ), "to_table_test_9", + skipped_dialects={"BODOSQL"}, ), id="to_table_test_9", ), @@ -7527,6 +7535,7 @@ def test_pipeline_e2e_errors( } ), "to_table_test_10", + skipped_dialects={"BODOSQL"}, ), id="to_table_test_10", ), @@ -7552,6 +7561,7 @@ def test_pipeline_e2e_errors( } ), "to_table_test_11", + skipped_dialects={"BODOSQL"}, ), id="to_table_test_11", ), @@ -7576,6 +7586,7 @@ def test_pipeline_e2e_errors( } ), "to_table_test_12", + skipped_dialects={"BODOSQL"}, ), id="to_table_test_12", ), @@ -7598,6 +7609,7 @@ def test_pipeline_e2e_errors( } ), "to_table_test_13", + skipped_dialects={"BODOSQL"}, ), id="to_table_test_13", ), @@ -7622,6 +7634,7 @@ def test_pipeline_e2e_errors( } ), "to_table_test_14", + skipped_dialects={"BODOSQL"}, ), id="to_table_test_14", ), @@ -7639,6 +7652,7 @@ def test_pipeline_e2e_errors( } ), "to_table_test_15", + skipped_dialects={"BODOSQL"}, ), id="to_table_test_15", ), @@ -7656,6 +7670,7 @@ def test_pipeline_e2e_errors( } ), "to_table_test_16", + skipped_dialects={"BODOSQL"}, ), id="to_table_test_16", ), @@ -7673,6 +7688,7 @@ def test_pipeline_e2e_errors( } ), "to_table_test_17", + skipped_dialects={"BODOSQL"}, ), id="to_table_test_17", ), @@ -7691,6 +7707,7 @@ def test_pipeline_e2e_errors( } ), "to_table_test_18", + skipped_dialects={"BODOSQL"}, ), id="to_table_test_18", ), @@ -7710,6 +7727,7 @@ def test_pipeline_e2e_errors( } ), "to_table_test_19", + skipped_dialects={"BODOSQL"}, ), id="to_table_test_19", ), @@ -7726,6 +7744,7 @@ def test_pipeline_e2e_errors( } ), "to_table_test_20", + skipped_dialects={"BODOSQL"}, ), id="to_table_test_20", ), @@ -7748,6 +7767,7 @@ def test_pipeline_e2e_errors( {"user_id": [1, 2, 3], "user_name": ["Alice", "Bob", "Charlie"]} ) }, + skipped_dialects={"BODOSQL"}, ), id="to_table_test_21", ), @@ -7783,6 +7803,7 @@ def test_pipeline_e2e_errors( {"pid": [10, 20], "product_name": ["Apple", "Banana"]} ) }, + skipped_dialects={"BODOSQL"}, ), id="to_table_test_22", ), @@ -7823,6 +7844,7 @@ def test_pipeline_e2e_errors( } ), "to_table_test_23", + skipped_dialects={"BODOSQL"}, ), id="to_table_test_23", ), @@ -7860,6 +7882,7 @@ def test_pipeline_e2e_errors( } ) }, + skipped_dialects={"BODOSQL"}, ), id="to_table_test_24", ), @@ -7905,6 +7928,7 @@ def test_pipeline_e2e_errors( } ), "to_table_test_25", + skipped_dialects={"BODOSQL"}, ), id="to_table_test_25", ), @@ -8011,9 +8035,6 @@ def test_pipeline_tpch_sql_to_table_all_dialects( """ db_context, graph = all_dialects_tpch_db_context - if db_context.dialect == DatabaseDialect.BODOSQL: - pytest.skip("TODO: (gh#500) to_table() is not yet implemented for BodoSQL") - table_prefix: str = get_table_prefix_for_dialect(db_context.dialect) test_data = tpch_custom_pipeline_to_table_test_data @@ -8041,6 +8062,7 @@ def test_pipeline_tpch_sql_to_table_all_dialects( "TPCH", lambda: pd.DataFrame(), "window_filter_order_1", + skipped_dialects={"BODOSQL"}, ), id="window_filter_order_1", ), @@ -8096,9 +8118,6 @@ def test_pipeline_to_table_ddl( db_context, graph = all_dialects_tpch_db_context - if db_context.dialect == DatabaseDialect.BODOSQL: - pytest.skip("TODO: (gh#500) to_table() is not yet implemented for BodoSQL") - table_prefix: str = get_table_prefix_for_dialect(db_context.dialect) # TEMP VIEWS (not tables) are not supported for Snowflake, MySQL, Postgres, and Oracle. diff --git a/tests/test_plan_refsols/explode_01.txt b/tests/test_plan_refsols/explode_01.txt index 519945caa..f7beed186 100644 --- a/tests/test_plan_refsols/explode_01.txt +++ b/tests/test_plan_refsols/explode_01.txt @@ -1,4 +1,4 @@ -ROOT(columns=[('region_name', region_name), ('nation_names', nation_names), ('nation_idx', nation_idx), ('nation_name', nation_name)], orderings=[(region_name):asc_first, (nation_idx):asc_first]) +ROOT(columns=[('region_name', region_name), ('nation_names', nation_names), ('nation_idx', nation_idx), ('nation_name', nation_name)], orderings=[(region_name):asc_first, (nation_name):asc_first]) EXPLODE(nation_names, value_name='nation_name', index_name='nation_idx', version='array', filtering=False, is_distinct=True, columns={'nation_idx': nation_idx, 'nation_name': nation_name, 'nation_names': nation_names, 'region_name': region_name}) JOIN(condition=t0.key == t1.region_key, type=INNER, cardinality=SINGULAR_ACCESS, reverse_cardinality=SINGULAR_ACCESS, columns={'nation_names': t1.nation_names, 'region_name': t0.region_name}) SCAN(table=tpch.REGION, columns={'key': r_regionkey, 'region_name': r_name}) diff --git a/tests/test_sql_refsols/color_q16_bodosql.sql b/tests/test_sql_refsols/color_q16_bodosql.sql index 7a4534db2..d28e14637 100644 --- a/tests/test_sql_refsols/color_q16_bodosql.sql +++ b/tests/test_sql_refsols/color_q16_bodosql.sql @@ -1,11 +1,11 @@ WITH _t1 AS ( SELECT - ARRAY_AGG(identname) AS listof_identname, chex, + ARRAY_AGG(identname) AS listof_identname, COUNT(*) AS n_rows FROM clrs GROUP BY - 2 + 1 ) SELECT chex AS hex_code, diff --git a/tests/test_sql_refsols/explode_01_databricks.sql b/tests/test_sql_refsols/explode_01_databricks.sql index 1b56dd853..fbea05896 100644 --- a/tests/test_sql_refsols/explode_01_databricks.sql +++ b/tests/test_sql_refsols/explode_01_databricks.sql @@ -17,4 +17,4 @@ JOIN _s1 AS _s1 CROSS JOIN LATERAL POSEXPLODE(_s1.nation_names) AS _s2(idx, val) ORDER BY 1, - 3 + 4 diff --git a/tests/test_sql_refsols/explode_01_duckdb.sql b/tests/test_sql_refsols/explode_01_duckdb.sql index 570bcb4ed..49ee40caa 100644 --- a/tests/test_sql_refsols/explode_01_duckdb.sql +++ b/tests/test_sql_refsols/explode_01_duckdb.sql @@ -21,4 +21,4 @@ CROSS JOIN LATERAL ( ) AS _s2(val, idx) ORDER BY 1 NULLS FIRST, - 3 NULLS FIRST + 4 NULLS FIRST diff --git a/tests/test_sql_refsols/explode_01_postgres.sql b/tests/test_sql_refsols/explode_01_postgres.sql index cb13f01ba..c15f5245b 100644 --- a/tests/test_sql_refsols/explode_01_postgres.sql +++ b/tests/test_sql_refsols/explode_01_postgres.sql @@ -17,4 +17,4 @@ JOIN _s1 AS _s1 CROSS JOIN LATERAL UNNEST(_s1.nation_names) WITH ORDINALITY AS _s2(val, idx) ORDER BY 1 NULLS FIRST, - 3 NULLS FIRST + 4 NULLS FIRST diff --git a/tests/test_sql_refsols/explode_01_snowflake.sql b/tests/test_sql_refsols/explode_01_snowflake.sql index cbbfc8b8d..808fff70a 100644 --- a/tests/test_sql_refsols/explode_01_snowflake.sql +++ b/tests/test_sql_refsols/explode_01_snowflake.sql @@ -17,4 +17,4 @@ JOIN _s1 AS _s1 CROSS JOIN LATERAL FLATTEN(_s1.nation_names) AS _s2(seq, key, path, index, value, this) ORDER BY 1 NULLS FIRST, - 3 NULLS FIRST + 4 NULLS FIRST diff --git a/tests/test_sql_refsols/explode_01_trino.sql b/tests/test_sql_refsols/explode_01_trino.sql index 0836d4cc6..740e28854 100644 --- a/tests/test_sql_refsols/explode_01_trino.sql +++ b/tests/test_sql_refsols/explode_01_trino.sql @@ -17,4 +17,4 @@ JOIN _s1 AS _s1 CROSS JOIN UNNEST(_s1.nation_names) WITH ORDINALITY AS _s2(val, idx) ORDER BY 1 NULLS FIRST, - 3 NULLS FIRST + 4 NULLS FIRST From c80670cf5101aa63d9dbc393be29284a90b3b949 Mon Sep 17 00:00:00 2001 From: knassre-bodo Date: Fri, 28 Aug 2026 10:18:45 -0700 Subject: [PATCH 41/44] Explode test fix [RUN CI][RUN DIALECTS] --- tests/test_pipeline_tpch_custom.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_pipeline_tpch_custom.py b/tests/test_pipeline_tpch_custom.py index 85213335d..db65f0178 100644 --- a/tests/test_pipeline_tpch_custom.py +++ b/tests/test_pipeline_tpch_custom.py @@ -4243,7 +4243,7 @@ "result = (" " array_data" " .EXPLODE(nation_names, 'exploded_nations', index_name='nation_idx', value_name='nation_name', version='array', filtering=False, is_distinct=True)" - " .CALCULATE(region_name, nation_names, nation_idx, nation_name)" + " .CALCULATE(region_name, nation_names, nation_name)" " .ORDER_BY(region_name, nation_name)" ")", "TPCH", From 4871bf91a95553bd47ed34ac7ea85ba3431ab3e7 Mon Sep 17 00:00:00 2001 From: knassre-bodo Date: Fri, 28 Aug 2026 10:22:37 -0700 Subject: [PATCH 42/44] Explode test fix [RUN CI][RUN DIALECTS] --- tests/test_pipeline_tpch_custom.py | 2 +- tests/test_plan_refsols/explode_01.txt | 2 +- tests/test_sql_refsols/explode_01_databricks.sql | 3 +-- tests/test_sql_refsols/explode_01_duckdb.sql | 3 +-- tests/test_sql_refsols/explode_01_postgres.sql | 3 +-- tests/test_sql_refsols/explode_01_snowflake.sql | 3 +-- tests/test_sql_refsols/explode_01_trino.sql | 3 +-- 7 files changed, 7 insertions(+), 12 deletions(-) diff --git a/tests/test_pipeline_tpch_custom.py b/tests/test_pipeline_tpch_custom.py index db65f0178..03d76e40e 100644 --- a/tests/test_pipeline_tpch_custom.py +++ b/tests/test_pipeline_tpch_custom.py @@ -4291,10 +4291,10 @@ "CANADA", "PERU", "UNITED STATES", + "CHINA", "INDIA", "INDONESIA", "JAPAN", - "CHINA", "VIETNAM", "FRANCE", "GERMANY", diff --git a/tests/test_plan_refsols/explode_01.txt b/tests/test_plan_refsols/explode_01.txt index f7beed186..05dee1cf4 100644 --- a/tests/test_plan_refsols/explode_01.txt +++ b/tests/test_plan_refsols/explode_01.txt @@ -1,4 +1,4 @@ -ROOT(columns=[('region_name', region_name), ('nation_names', nation_names), ('nation_idx', nation_idx), ('nation_name', nation_name)], orderings=[(region_name):asc_first, (nation_name):asc_first]) +ROOT(columns=[('region_name', region_name), ('nation_names', nation_names), ('nation_name', nation_name)], orderings=[(region_name):asc_first, (nation_name):asc_first]) EXPLODE(nation_names, value_name='nation_name', index_name='nation_idx', version='array', filtering=False, is_distinct=True, columns={'nation_idx': nation_idx, 'nation_name': nation_name, 'nation_names': nation_names, 'region_name': region_name}) JOIN(condition=t0.key == t1.region_key, type=INNER, cardinality=SINGULAR_ACCESS, reverse_cardinality=SINGULAR_ACCESS, columns={'nation_names': t1.nation_names, 'region_name': t0.region_name}) SCAN(table=tpch.REGION, columns={'key': r_regionkey, 'region_name': r_name}) diff --git a/tests/test_sql_refsols/explode_01_databricks.sql b/tests/test_sql_refsols/explode_01_databricks.sql index fbea05896..f12be5383 100644 --- a/tests/test_sql_refsols/explode_01_databricks.sql +++ b/tests/test_sql_refsols/explode_01_databricks.sql @@ -9,7 +9,6 @@ WITH _s1 AS ( SELECT region.r_name AS region_name, _s1.nation_names, - _s2.idx AS nation_idx, _s2.val AS nation_name FROM tpch.region AS region JOIN _s1 AS _s1 @@ -17,4 +16,4 @@ JOIN _s1 AS _s1 CROSS JOIN LATERAL POSEXPLODE(_s1.nation_names) AS _s2(idx, val) ORDER BY 1, - 4 + 3 diff --git a/tests/test_sql_refsols/explode_01_duckdb.sql b/tests/test_sql_refsols/explode_01_duckdb.sql index 49ee40caa..af4bf9607 100644 --- a/tests/test_sql_refsols/explode_01_duckdb.sql +++ b/tests/test_sql_refsols/explode_01_duckdb.sql @@ -9,7 +9,6 @@ WITH _s1 AS ( SELECT region.r_name AS region_name, _s1.nation_names, - _s2.idx AS nation_idx, _s2.val AS nation_name FROM tpch.region AS region JOIN _s1 AS _s1 @@ -21,4 +20,4 @@ CROSS JOIN LATERAL ( ) AS _s2(val, idx) ORDER BY 1 NULLS FIRST, - 4 NULLS FIRST + 3 NULLS FIRST diff --git a/tests/test_sql_refsols/explode_01_postgres.sql b/tests/test_sql_refsols/explode_01_postgres.sql index c15f5245b..01ca866b4 100644 --- a/tests/test_sql_refsols/explode_01_postgres.sql +++ b/tests/test_sql_refsols/explode_01_postgres.sql @@ -9,7 +9,6 @@ WITH _s1 AS ( SELECT region.r_name AS region_name, _s1.nation_names, - _s2.idx - 1 AS nation_idx, _s2.val AS nation_name FROM tpch.region AS region JOIN _s1 AS _s1 @@ -17,4 +16,4 @@ JOIN _s1 AS _s1 CROSS JOIN LATERAL UNNEST(_s1.nation_names) WITH ORDINALITY AS _s2(val, idx) ORDER BY 1 NULLS FIRST, - 4 NULLS FIRST + 3 NULLS FIRST diff --git a/tests/test_sql_refsols/explode_01_snowflake.sql b/tests/test_sql_refsols/explode_01_snowflake.sql index 808fff70a..2d49fe9b4 100644 --- a/tests/test_sql_refsols/explode_01_snowflake.sql +++ b/tests/test_sql_refsols/explode_01_snowflake.sql @@ -9,7 +9,6 @@ WITH _s1 AS ( SELECT region.r_name AS region_name, _s1.nation_names, - _s2.index AS nation_idx, _s2.value AS nation_name FROM tpch.region AS region JOIN _s1 AS _s1 @@ -17,4 +16,4 @@ JOIN _s1 AS _s1 CROSS JOIN LATERAL FLATTEN(_s1.nation_names) AS _s2(seq, key, path, index, value, this) ORDER BY 1 NULLS FIRST, - 4 NULLS FIRST + 3 NULLS FIRST diff --git a/tests/test_sql_refsols/explode_01_trino.sql b/tests/test_sql_refsols/explode_01_trino.sql index 740e28854..2a128b459 100644 --- a/tests/test_sql_refsols/explode_01_trino.sql +++ b/tests/test_sql_refsols/explode_01_trino.sql @@ -9,7 +9,6 @@ WITH _s1 AS ( SELECT region.r_name AS region_name, _s1.nation_names, - _s2.idx - 1 AS nation_idx, _s2.val AS nation_name FROM tpch.region AS region JOIN _s1 AS _s1 @@ -17,4 +16,4 @@ JOIN _s1 AS _s1 CROSS JOIN UNNEST(_s1.nation_names) WITH ORDINALITY AS _s2(val, idx) ORDER BY 1 NULLS FIRST, - 4 NULLS FIRST + 3 NULLS FIRST From af9b38c280eae5034a5e935487ad1f5d9d1fa27f Mon Sep 17 00:00:00 2001 From: knassre-bodo Date: Wed, 9 Sep 2026 02:31:49 -0700 Subject: [PATCH 43/44] Revisions [RUN CI][RUN DIALECTS] --- documentation/dsl.md | 6 ++-- documentation/metadata.md | 18 ++++++++-- .../database_connectors/database_connector.py | 6 +++- pydough/errors/pydough_error_builder.py | 4 +-- .../expression_operators/README.md | 1 + .../registered_expression_operators.py | 3 +- .../type_inference/__init__.py | 2 ++ .../type_inference/expression_type_deducer.py | 34 ++++++++++++++++++- pydough/qdag/collections/explode.py | 7 ++++ pydough/qdag/node_builder.py | 2 +- .../base_transform_bindings.py | 4 +-- pydough/unqualified/unqualified_node.py | 2 +- pydough/utilities/explode_spec.py | 4 +-- .../simple_pydough_functions.py | 8 ++--- tests/test_qualification_errors.py | 5 +++ 15 files changed, 85 insertions(+), 21 deletions(-) diff --git a/documentation/dsl.md b/documentation/dsl.md index 6c278a589..661e7a399 100644 --- a/documentation/dsl.md +++ b/documentation/dsl.md @@ -1612,13 +1612,13 @@ thesaurus.CALCULATE(word) ``` The result would be the following table: -| name | syn_idx | synonym | +| word | syn_idx | synonym | |---------|---------|--------------| | 'wise' | 0 | 'sage' | | 'wise' | 1 | 'insightful' | | 'wise' | 2 | 'keen' | -| 'old' | 0 | 'elderly | -| 'old' | 1 | 'ancient | +| 'old' | 0 | 'elderly' | +| 'old' | 1 | 'ancient' | | 'large' | 0 | 'big' | **Bad Example #1**: Missing the `name`. diff --git a/documentation/metadata.md b/documentation/metadata.md index e6230bf5b..e8b6cf826 100644 --- a/documentation/metadata.md +++ b/documentation/metadata.md @@ -466,12 +466,12 @@ Below are several examples the JSON for such verifiers: The JSON for a function deducer, used in the `output signature` field of a function definition, specifies the rules for determining the output type of a call to the function in terms of its input arguments. If a verifier is not provided, the default assumption is that the function call outputs an expression of type `"unknown"`. -Each deducer has a mandatory string field `type` specifying what kind of verifier it is. The currently supported values are `"constant"` and `"select argument"`. +Each deducer has a mandatory string field `type` specifying what kind of verifier it is. The currently supported values are `"constant"`, `"select argument"` and `"array of"`. ### Function Deducer Type: Constant -Function deducers of this type have a type string of `"constant"` and correspond to a function call that always returns the same type. Verifiers of this type have the following additional key-value pairs in their metadata JSON object: +Function deducers of this type have a type string of `"constant"` and correspond to a function call that always returns the same type. Deducers of this type have the following additional key-value pairs in their metadata JSON object: - `value` (required): a type string ([see here for more information](#pydough-type-strings)) indicating what type the function always returns. @@ -484,7 +484,7 @@ Below are several examples the JSON for such deducers: ### Function Deducer Type: Select Argument -Function deducers of this type have a type string of `"select argument"` and correspond to a function call that always returns a value of the same type as a specific argument. Verifiers of this type have the following additional key-value pairs in their metadata JSON object: +Function deducers of this type have a type string of `"select argument"` and correspond to a function call that always returns a value of the same type as a specific argument. Deducers of this type have the following additional key-value pairs in their metadata JSON object: - `value` (required): a non-negative integer indicating which input argument to the function call should determine the output type of the function when called. @@ -493,6 +493,18 @@ Below are several examples the JSON for such deducers: - Returns the type of the first argument: `{"type": "select argument", "value": 0}` - Returns the type of the second argument: `{"type": "select argument", "value": 1}` + +### Function Deducer Type: Array Of + +Function deducers of this type have a type string of `"array of"` and correspond to a function call that returns an array type. Deducers of this type have the following additional key-value pairs in their metadata JSON object: + +- `element type` (required): a JSON object containing the specification for another function deducer containing the element type of the array. + +Below are several examples the JSON for such deducers: + +- Returns an array of strings: `{"type": "array of", "element type": {"type": "constant", "value": "string"}}` +- Returns an array where the elements have the same type as the first argument: `{"type": "array of", "element type": {"type": "select argument", "value": 0}}` + ## PyDough Type Strings diff --git a/pydough/database_connectors/database_connector.py b/pydough/database_connectors/database_connector.py index 620e8956a..e4c029a5a 100644 --- a/pydough/database_connectors/database_connector.py +++ b/pydough/database_connectors/database_connector.py @@ -108,8 +108,12 @@ def execute_query_df(self, sql: str) -> pd.DataFrame: # TODO: (gh #175) enable typed DataFrames. data = self.cursor.fetchall() pd_table = pd.DataFrame(data, columns=column_names) + # Parse all semi-structured columns from JSON strings into native + # Python types. for idx in semi_structured_cols: - pd_table.iloc[:, idx] = pd_table.iloc[:, idx].apply(json.loads) + pd_table.iloc[:, idx] = pd_table.iloc[:, idx].apply( + lambda s: None if s is None else json.loads(s) + ) return pd_table except Exception as e: print(f"ERROR WHILE EXECUTING QUERY:\n{sql}") diff --git a/pydough/errors/pydough_error_builder.py b/pydough/errors/pydough_error_builder.py index 0733c45da..ae92c5a17 100644 --- a/pydough/errors/pydough_error_builder.py +++ b/pydough/errors/pydough_error_builder.py @@ -2,7 +2,7 @@ Definition of the base class for creating exceptions in PyDough. """ -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Union from pydough.errors import ( PyDoughException, @@ -256,7 +256,7 @@ def sql_call_conversion_error( ) def sql_call_dialect_unsupported( - self, operator: "PyDoughOperator", dialect: str + self, operator: Union["PyDoughOperator", "str"], dialect: str ) -> PyDoughException: """ Creates an exception for when a SQL dialect does not allow converting diff --git a/pydough/pydough_operators/expression_operators/README.md b/pydough/pydough_operators/expression_operators/README.md index f74f4bb89..6827db3ca 100644 --- a/pydough/pydough_operators/expression_operators/README.md +++ b/pydough/pydough_operators/expression_operators/README.md @@ -150,6 +150,7 @@ These functions can be called on plural data to aggregate it into a singular exp - `SAMPLE_STD`: returns the sample standard deviation of the values of a plural expression. - `POPULATION_VAR`: returns the population variance of the values of a plural expression. - `POPULATION_STD`: returns the population standard deviation of the values of a plural expression. +- `LISTOF`: returns an array of the elements within the plural expression. ##### Collection Aggregations diff --git a/pydough/pydough_operators/expression_operators/registered_expression_operators.py b/pydough/pydough_operators/expression_operators/registered_expression_operators.py index c7b55ff50..9afda0a2d 100644 --- a/pydough/pydough_operators/expression_operators/registered_expression_operators.py +++ b/pydough/pydough_operators/expression_operators/registered_expression_operators.py @@ -96,6 +96,7 @@ from pydough.pydough_operators.type_inference import ( AllowAny, + ArrayOfType, ConstantType, RequireArgRange, RequireCollection, @@ -170,7 +171,7 @@ "QUANTILE", True, RequireNumArgs(2), ConstantType(NumericType()) ) LISTOF = ExpressionFunctionOperator( - "LISTOF", True, RequireNumArgs(1), SelectArgumentType(0) + "LISTOF", True, RequireNumArgs(1), ArrayOfType(SelectArgumentType(0)) ) POWER = ExpressionFunctionOperator( "POWER", False, RequireNumArgs(2), ConstantType(NumericType()) diff --git a/pydough/pydough_operators/type_inference/__init__.py b/pydough/pydough_operators/type_inference/__init__.py index 1f7f1b3b9..24672b96a 100644 --- a/pydough/pydough_operators/type_inference/__init__.py +++ b/pydough/pydough_operators/type_inference/__init__.py @@ -5,6 +5,7 @@ __all__ = [ "AllowAny", + "ArrayOfType", "ConstantType", "ExpressionTypeDeducer", "RequireArgRange", @@ -19,6 +20,7 @@ ] from .expression_type_deducer import ( + ArrayOfType, ConstantType, ExpressionTypeDeducer, SelectArgumentType, diff --git a/pydough/pydough_operators/type_inference/expression_type_deducer.py b/pydough/pydough_operators/type_inference/expression_type_deducer.py index af83af0f2..e8550869f 100644 --- a/pydough/pydough_operators/type_inference/expression_type_deducer.py +++ b/pydough/pydough_operators/type_inference/expression_type_deducer.py @@ -3,6 +3,7 @@ """ __all__ = [ + "ArrayOfType", "ConstantType", "ExpressionTypeDeducer", "SelectArgumentType", @@ -16,9 +17,10 @@ NoExtraKeys, PyDoughMetadataException, extract_integer, + extract_object, extract_string, ) -from pydough.types import PyDoughType, UnknownType, parse_type_from_string +from pydough.types import ArrayType, PyDoughType, UnknownType, parse_type_from_string class ExpressionTypeDeducer(ABC): @@ -87,6 +89,26 @@ def infer_return_type(self, args: list[Any]) -> PyDoughType: return self.data_type +class ArrayOfType(ExpressionTypeDeducer): + """ + Type deduction implementation class that always returns an array type + of a specific PyDough type. + """ + + def __init__(self, inner_builder: ExpressionTypeDeducer): + self._inner_builder: ExpressionTypeDeducer = inner_builder + + @property + def inner_builder(self) -> ExpressionTypeDeducer: + """ + The inner type deducer used to determine the array element type. + """ + return self._inner_builder + + def infer_return_type(self, args: list[Any]) -> PyDoughType: + return ArrayType(self.inner_builder.infer_return_type(args)) + + def build_deducer_from_json(json_data: dict[str, Any] | None) -> ExpressionTypeDeducer: """ Builds a type deducer from a JSON object. @@ -137,5 +159,15 @@ def build_deducer_from_json(json_data: dict[str, Any] | None) -> ExpressionTypeD ) return SelectArgumentType(arg_idx) + # Select argument deducer type. + case "array of": + NoExtraKeys({"type", "element type"}).verify( + json_data, "array of deducer JSON metadata" + ) + inner_dict: dict[str, Any] = extract_object( + json_data, "element type", "array of deducer JSON data" + ) + return ArrayOfType(build_deducer_from_json(inner_dict)) + case _: raise PyDoughMetadataException(f"Unknown deducer type: {deducer_type!r}") diff --git a/pydough/qdag/collections/explode.py b/pydough/qdag/collections/explode.py index 3252e89c7..3810feb00 100644 --- a/pydough/qdag/collections/explode.py +++ b/pydough/qdag/collections/explode.py @@ -45,6 +45,13 @@ def __init__( raise PyDoughQDAGException( f"Cannot use {explode_spec.index_name!r} as the `index_name` for EXPLODE because it is already a term in the ancestor context" ) + if ( + explode_spec.index_name is not None + and explode_spec.index_name == explode_spec.value_name + ): + raise PyDoughQDAGException( + f"Cannot use {explode_spec.index_name!r} as the `index_name` for EXPLODE because it is the same as the `value_name`" + ) self._name: str = name self._data: PyDoughExpressionQDAG = data self._explode_spec: ExplodeSpec = explode_spec diff --git a/pydough/qdag/node_builder.py b/pydough/qdag/node_builder.py index 9fea86f81..b835dab8b 100644 --- a/pydough/qdag/node_builder.py +++ b/pydough/qdag/node_builder.py @@ -409,7 +409,7 @@ def build_explode( data: PyDoughExpressionQDAG, name: str, explode_spec: ExplodeSpec, - ): + ) -> Explode: """ Creates an EXPLODE instance. diff --git a/pydough/sqlglot/transform_bindings/base_transform_bindings.py b/pydough/sqlglot/transform_bindings/base_transform_bindings.py index 9498158e6..0d8779b05 100644 --- a/pydough/sqlglot/transform_bindings/base_transform_bindings.py +++ b/pydough/sqlglot/transform_bindings/base_transform_bindings.py @@ -2660,8 +2660,8 @@ def generate_dataframe_array_expression( Returns: A SQLGlotExpression representing the array of items. """ - raise NotImplementedError( - f"Array types are not currently supported in dialect {self._visitor._expr_visitor._dialect.name}." + raise self._visitor._session.error_builder.sql_call_dialect_unsupported( + "LITERAL ARRAY", self._visitor._session.database.dialect.name ) def generate_dataframe_item_dialect_expression( diff --git a/pydough/unqualified/unqualified_node.py b/pydough/unqualified/unqualified_node.py index d3f101773..dadaedec4 100644 --- a/pydough/unqualified/unqualified_node.py +++ b/pydough/unqualified/unqualified_node.py @@ -446,7 +446,7 @@ def EXPLODE( delimiter: str | None = None, filtering: bool = True, is_distinct: bool = False, - ): + ) -> "UnqualifiedNode": """ Method used to create an EXPLODE node, transforming the existing collection by exploding each row into multiple rows based on the values diff --git a/pydough/utilities/explode_spec.py b/pydough/utilities/explode_spec.py index 3470ff3a7..91161620a 100644 --- a/pydough/utilities/explode_spec.py +++ b/pydough/utilities/explode_spec.py @@ -19,7 +19,7 @@ class ExplodeSpec: is_distinct: bool @property - def arg_list_string(self): + def arg_list_string(self) -> str: args: list[str] = [] args.append(self.value_name) if self.index_name is not None: @@ -32,7 +32,7 @@ def arg_list_string(self): return ", ".join(args) @property - def keyword_arg_string(self): + def keyword_arg_string(self) -> str: kwargs: list[str] = [] kwargs.append(f"value_name={self.value_name!r}") if self.index_name is not None: diff --git a/tests/test_pydough_functions/simple_pydough_functions.py b/tests/test_pydough_functions/simple_pydough_functions.py index 477d767c1..7023606b1 100644 --- a/tests/test_pydough_functions/simple_pydough_functions.py +++ b/tests/test_pydough_functions/simple_pydough_functions.py @@ -102,8 +102,8 @@ def good_explode_04(): value_name="v", index_name="i", version="array", - filtering=True, - is_distinct=False, + filtering=False, + is_distinct=True, ) @@ -154,8 +154,8 @@ def good_explode_08(): index_name="i", version="string", delimiter=" ", - filtering=True, - is_distinct=False, + filtering=False, + is_distinct=True, ) diff --git a/tests/test_qualification_errors.py b/tests/test_qualification_errors.py index 3641d2df3..b2b0b3289 100644 --- a/tests/test_qualification_errors.py +++ b/tests/test_qualification_errors.py @@ -310,6 +310,11 @@ "Cannot use 'key' as the `index_name` for EXPLODE because it is already a term in the ancestor context", id="bad_explode_22", ), + pytest.param( + "result = nations.EXPLODE(name, 'names', value_name='v', index_name='v', version='string', delimiter='#')", + "Cannot use 'v' as the `index_name` for EXPLODE because it is the same as the `value_name`", + id="bad_explode_23", + ), ], ) def test_qualify_error( From c7f99797f14d00d067a0b9c0370687f883abf089 Mon Sep 17 00:00:00 2001 From: knassre-bodo Date: Wed, 9 Sep 2026 10:09:56 -0700 Subject: [PATCH 44/44] Regular test fixes [RUN CI] --- tests/test_pipeline_tpch_custom.py | 4 +++- tests/test_qualification.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_pipeline_tpch_custom.py b/tests/test_pipeline_tpch_custom.py index 03d76e40e..11f26d555 100644 --- a/tests/test_pipeline_tpch_custom.py +++ b/tests/test_pipeline_tpch_custom.py @@ -7181,7 +7181,9 @@ def test_pipeline_e2e_simple_week( pytest.param( dataframe_collection_bad_5, None, - re.escape("Array types are not currently supported in dialect SQLITE"), + re.escape( + "Cannot convert function LITERAL ARRAY to SQL using dialect SQLITE" + ), id="dataframe_collection_bad_5", ), pytest.param( diff --git a/tests/test_qualification.py b/tests/test_qualification.py index 3ff336008..3151ae857 100644 --- a/tests/test_qualification.py +++ b/tests/test_qualification.py @@ -1000,7 +1000,7 @@ """ ──┬─ TPCH └─┬─ TableCollection[nations] - └─── Explode[name, name='exploded_nation', value_name='v', index_name='i', version='array', filtering=True, is_distinct=False] + └─── Explode[name, name='exploded_nation', value_name='v', index_name='i', version='array', filtering=False, is_distinct=True] """, id="good_explode_04", ), @@ -1036,7 +1036,7 @@ """ ──┬─ TPCH └─┬─ TableCollection[nations] - └─── Explode[name, name='exploded_nation', value_name='v', index_name='i', version='string', delimiter=' ', filtering=True, is_distinct=False] + └─── Explode[name, name='exploded_nation', value_name='v', index_name='i', version='string', delimiter=' ', filtering=False, is_distinct=True] """, id="good_explode_08", ),