diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java index 6888670993..ed3905f33b 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java @@ -54,7 +54,7 @@ public class CQConceptConverter implements NodeConverter { public CQConceptConverter() { this.connectorCTEs = List.of( new PreprocessingCte(), - new EventFilterCte(), + // new EventFilterCte(), new AggregationSelectCte(), new JoinBranchesCte(), new AggregationFilterCte() diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptCteStep.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptCteStep.java index 01ad5d1593..191dadec6f 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptCteStep.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptCteStep.java @@ -13,8 +13,7 @@ public enum ConceptCteStep implements CteStep { // connector PREPROCESSING("preprocessing", null), - EVENT_FILTER("event_filter", PREPROCESSING), - AGGREGATION_SELECT("group_select", EVENT_FILTER), + AGGREGATION_SELECT("group_select", PREPROCESSING), JOIN_BRANCHES("join_branches", AGGREGATION_SELECT), AGGREGATION_FILTER("group_filter", JOIN_BRANCHES), @@ -27,7 +26,6 @@ public enum ConceptCteStep implements CteStep { public static final Set MANDATORY_STEPS = Set.of( PREPROCESSING, - EVENT_FILTER, AGGREGATION_SELECT, JOIN_BRANCHES, AGGREGATION_FILTER diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/EventFilterCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/EventFilterCte.java deleted file mode 100644 index 18551f789d..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/EventFilterCte.java +++ /dev/null @@ -1,130 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement.concept; - -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.stream.Stream; - -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.Selects; -import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; -import com.bakdata.conquery.sql.conversion.model.aggregator.SumSqlAggregator; -import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; -import com.bakdata.conquery.sql.conversion.model.select.ConnectorSqlSelects; -import com.bakdata.conquery.sql.conversion.model.select.ExtractingSqlSelect; -import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; -import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; -import com.google.common.base.Preconditions; -import org.jooq.Condition; - -class EventFilterCte extends ConnectorCte { - - @Override - public QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { - List conditions = new ArrayList<>(); - - if (tableContext.getIds().getSecondaryId().isPresent()) { - conditions.add(tableContext.getIds().getSecondaryId().get().isNotNull()); - } - - conditions.addAll(collectEventFilterConditions(tableContext)); - - return QueryStep.builder() - .selects(collectSelects(tableContext)) - .conditions(conditions); - } - - @Override - public ConceptCteStep cteStep() { - return ConceptCteStep.EVENT_FILTER; - } - - private Selects collectSelects(CQTableContext tableContext) { - - String predecessorTableName = tableContext.getPrevious().getCteName(); - Selects predecessorSelects = tableContext.getPrevious().getQualifiedSelects(); - - SqlIdColumns ids = predecessorSelects.getIds(); - Optional validityDate = predecessorSelects.getValidityDate(); - Optional stratificationDate = predecessorSelects.getStratificationDate(); - - List eventFilterSelects = - tableContext.allSqlSelects().stream() - .flatMap(sqlSelects -> collectSelects(sqlSelects).stream()) - .flatMap(sqlSelect -> referenceRequiredColumns(sqlSelect, predecessorTableName)) - .toList(); - - return Selects.builder() - .ids(ids) - .validityDate(validityDate) - .stratificationDate(stratificationDate) - .sqlSelects(eventFilterSelects) - .build(); - } - - /** - * Collects the columns required in {@link ConceptCteStep#AGGREGATION_SELECT}, the optional connector column, but also columns additional tables require - * (like the ones created by the {@link SumSqlAggregator}) when distinct-by columns are present. An additional predecessor can contain an N-ary tree of - * predecessors itself (like all {@link QueryStep}s), so we want to look for the deepest preceding QueryStep leafs and collect their - * {@link ConnectorSqlSelects}, because they expect this CTE to contain all their {@link SqlSelect#requiredColumns()}. - */ - private static List collectSelects(ConnectorSqlSelects sqlSelects) { - return Stream.concat( - sqlSelects.getConnectorColumn().stream(), - Stream.concat( - sqlSelects.getAggregationSelects().stream(), - sqlSelects.getAdditionalPredecessor().map(EventFilterCte::collectDeepestPredecessorsColumns).orElse(Stream.empty()) - ) - ) - .toList(); - } - - /** - * Recursively looks for the deepest predecessors of the given additional predecessor and collects their {@link SqlSelect}. - */ - private static Stream collectDeepestPredecessorsColumns(QueryStep additionalPredecessor) { - if (additionalPredecessor.getPredecessors().isEmpty()) { - return Stream.concat( - additionalPredecessor.getSelects().getSqlSelects().stream(), - additionalPredecessor.getGroupBy().stream().map(FieldWrapper::new) // some required columns may just be referenced in a GROUP BY statement - ); - } - return additionalPredecessor.getPredecessors().stream().flatMap(EventFilterCte::collectDeepestPredecessorsColumns); - } - - private static Stream> referenceRequiredColumns(SqlSelect sqlSelect, String predecessorTableName) { - return sqlSelect.requiredColumns().stream().map(column -> new ExtractingSqlSelect<>(predecessorTableName, column, Object.class)); - } - - private static List collectEventFilterConditions(CQTableContext tableContext) { - - List eventFilterConditions = tableContext.getSqlFilters().stream() - .flatMap(conceptFilter -> conceptFilter.getWhereClauses().getEventFilters().stream()) - .map(WhereCondition::condition) - .toList(); - - if (tableContext.getConversionContext().isWithStratification()) { - return addStratificationCondition(eventFilterConditions, tableContext); - } - return eventFilterConditions; - } - - private static List addStratificationCondition(List eventFilterConditions, CQTableContext tableContext) { - Selects previousSelects = tableContext.getPrevious().getQualifiedSelects(); - Preconditions.checkArgument( - previousSelects.getStratificationDate().isPresent() && previousSelects.getValidityDate().isPresent(), - "Can't apply stratification for table %s".formatted(tableContext.getConnectorTables().getRootTable()) - ); - - // we filter every entry where stratification date range and validity date range do not overlap - SqlFunctionProvider functionProvider = tableContext.getFunctionProvider(); - ColumnDateRange stratificationDate = previousSelects.getStratificationDate().get(); - ColumnDateRange validityDate = previousSelects.getValidityDate().get(); - Condition stratificationCondition = functionProvider.dateRestriction(stratificationDate, validityDate); - - return Stream.concat(Stream.of(stratificationCondition), eventFilterConditions.stream()).toList(); - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java index 3ced541d97..bcdb3771bc 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java @@ -3,14 +3,17 @@ import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.stream.Stream; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; +import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; +import com.google.common.base.Preconditions; import org.jooq.Condition; import org.jooq.Record; import org.jooq.Table; @@ -19,6 +22,35 @@ class PreprocessingCte extends ConnectorCte { + public static List collectEventFilterConditions(CQTableContext tableContext) { + + List eventFilterConditions = tableContext.getSqlFilters().stream() + .flatMap(conceptFilter -> conceptFilter.getWhereClauses().getEventFilters().stream()) + .map(WhereCondition::condition) + .toList(); + + if (tableContext.getConversionContext().isWithStratification()) { + return addStratificationCondition(eventFilterConditions, tableContext); + } + return eventFilterConditions; + } + + public static List addStratificationCondition(List eventFilterConditions, CQTableContext tableContext) { + Selects previousSelects = tableContext.getPrevious().getQualifiedSelects(); + Preconditions.checkArgument( + previousSelects.getStratificationDate().isPresent() && previousSelects.getValidityDate().isPresent(), + "Can't apply stratification for table %s".formatted(tableContext.getConnectorTables().getRootTable()) + ); + + // we filter every entry where stratification date range and validity date range do not overlap + SqlFunctionProvider functionProvider = tableContext.getFunctionProvider(); + ColumnDateRange stratificationDate = previousSelects.getStratificationDate().get(); + ColumnDateRange validityDate = previousSelects.getValidityDate().get(); + Condition stratificationCondition = functionProvider.dateRestriction(stratificationDate, validityDate); + + return Stream.concat(Stream.of(stratificationCondition), eventFilterConditions.stream()).toList(); + } + @Override public ConceptCteStep cteStep() { return ConceptCteStep.PREPROCESSING; @@ -36,6 +68,7 @@ public QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { .validityDate(Optional.of(tableContext.getValidityDate())) .sqlSelects(forPreprocessing) .build(); + // all where clauses that don't require any preprocessing (connector/child conditions) List conditions = new ArrayList<>(); @@ -45,15 +78,20 @@ public QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { } } + conditions.addAll(collectEventFilterConditions(tableContext)); - QueryStep.QueryStepBuilder builder = QueryStep.builder() - .selects(preprocessingSelects) - .conditions(conditions); + if (tableContext.getIds().getSecondaryId().isPresent()) { + conditions.add(tableContext.getIds().getSecondaryId().get().isNotNull()); + } if (tableContext.getConversionContext().isWithStratification()) { return joinWithStratificationTable(forPreprocessing, conditions, tableContext); } + QueryStep.QueryStepBuilder builder = QueryStep.builder() + .selects(preprocessingSelects) + .conditions(conditions); + TableLike rootTable = QueryStep.toTableLike(tableContext.getConnectorTables().getPredecessor(ConceptCteStep.PREPROCESSING)); return builder.fromTable(rootTable); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java index dba9c97022..1452cb1256 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java @@ -91,7 +91,7 @@ private static TablePathInfo collectConnectorTables(CQConcept cqConcept, CQTable // interval packing requiredw tableInfo.setContainsIntervalPacking(true); - tableInfo.addMappings(IntervalPackingCteStep.getMappings(EVENT_FILTER, context.getDialectBundle())); + tableInfo.addMappings(IntervalPackingCteStep.getMappings(PREPROCESSING, context.getDialectBundle())); // validity date propagation not necessary if (!cqConcept.isAggregateEventDates()) { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDistinctSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDistinctSelectConverter.java index 23b6045f5a..202f67e220 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDistinctSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/clickhouse/ClickhouseDistinctSelectConverter.java @@ -24,7 +24,7 @@ public ConnectorSqlSelects connectorSelect(DistinctSelect distinctSelect, Select SingleColumnSqlSelect preprocessingSelect = MappableSingleColumnSelect.getSubstringSelect(distinctSelect.getColumn().get(), distinctSelect.getSubstringRange(), selectContext, alias); - String eventFilterTable = selectContext.getTables().cteName(ConceptCteStep.EVENT_FILTER); + String eventFilterTable = selectContext.getTables().cteName(ConceptCteStep.PREPROCESSING); SingleColumnSqlSelect qualified = preprocessingSelect.qualify(eventFilterTable); FieldWrapper grouped = diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/README.md b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/README.md index d5895c9456..4fb177e9eb 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/README.md +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/README.md @@ -375,7 +375,7 @@ from "vers_stamm" on "full_stratification"."primary_id" = "vers_stamm"."pid" ``` -2. In the `EVENT_FILTER` step, we filter all entries where the stratification range and the subjects validity date do +2. In the `PREPROCESSING` step, we filter all entries where the stratification range and the subjects validity date do not overlap. This is important because we only want to compute aggregations for those ranges that satisfy this condition. diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java index 8f438e4654..e0ebf555de 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java @@ -40,7 +40,7 @@ private static CommonAggregationSelect createSingleDateColumnAggregatio ExtractingSqlSelect rootSelect = new ExtractingSqlSelect<>(tables.getRootTable(), countColumn.getName(), Date.class); - Field qualifiedRootSelect = rootSelect.qualify(tables.cteName(ConceptCteStep.EVENT_FILTER)).select(); + Field qualifiedRootSelect = rootSelect.qualify(tables.cteName(ConceptCteStep.PREPROCESSING)).select(); FieldWrapper countQuartersAggregation = new FieldWrapper<>(DSL.nullif(DSL.countDistinct(functionProvider.yearQuarter(qualifiedRootSelect)), 0).as(alias), countColumn.getName()); @@ -73,7 +73,7 @@ private static CommonAggregationSelect sumQuarterCount( Field quarterCount = calcQuarterCount(quarterStart, nextQuarterStart, alias, functionProvider); FieldWrapper quarterCountWrapper = new FieldWrapper<>(quarterCount); - Field qualifiedQuarterCount = quarterCountWrapper.qualify(tables.cteName(ConceptCteStep.EVENT_FILTER)).select(); + Field qualifiedQuarterCount = quarterCountWrapper.qualify(tables.cteName(ConceptCteStep.PREPROCESSING)).select(); FieldWrapper quarterCountAggregation = new FieldWrapper<>(DSL.nullif(DSL.sum(qualifiedQuarterCount), BigDecimal.ZERO).as(alias)); return CommonAggregationSelect.builder().rootSelect(quarterCountWrapper).groupBy(quarterCountAggregation).build(); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java index b46f250ed4..e462cb3fb7 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java @@ -16,12 +16,12 @@ import com.bakdata.conquery.sql.conversion.model.select.*; import org.jooq.Condition; import org.jooq.Field; -import org.jooq.impl.DSL; import java.sql.Date; import java.time.LocalDate; import java.time.temporal.ChronoUnit; +import static org.jooq.impl.DSL.*; import static org.jooq.impl.DSL.field; public class DateDistanceSqlAggregator implements SelectConverter, FilterConverter { @@ -33,16 +33,18 @@ public ConnectorSqlSelects connectorSelect(DateDistanceSelect select, SelectCont String alias = selectContext.getNameGenerator().selectName(select); ConnectorSqlTables tables = selectContext.getTables(); - FieldWrapper dateDistanceSelect = createDateDistanceSelect(column, alias, select.getTimeUnit(), tables, selectContext.getConversionContext()); + Field dateDistanceSelect = createDateDistanceSelect(column, select.getTimeUnit(), tables, selectContext.getConversionContext()) + .as(alias); + FieldWrapper dateDistanceWrapper = new FieldWrapper<>(dateDistanceSelect); - Field qualifiedDateDistance = dateDistanceSelect.qualify(tables.getPredecessor(ConceptCteStep.AGGREGATION_SELECT)).select(); - FieldWrapper minDateDistance = new FieldWrapper<>(DSL.min(qualifiedDateDistance).as(alias)); + Field qualifiedDateDistance = dateDistanceWrapper.qualify(tables.getPredecessor(ConceptCteStep.AGGREGATION_SELECT)).select(); + FieldWrapper minDateDistance = new FieldWrapper<>(min(qualifiedDateDistance).as(alias)); String finalPredecessor = tables.getPredecessor(ConceptCteStep.AGGREGATION_FILTER); ExtractingSqlSelect finalSelect = minDateDistance.qualify(finalPredecessor); return ConnectorSqlSelects.builder() - .preprocessingSelect(dateDistanceSelect) + .preprocessingSelect(dateDistanceWrapper) .aggregationSelect(minDateDistance) .finalSelect(finalSelect) .build(); @@ -52,48 +54,36 @@ public ConnectorSqlSelects connectorSelect(DateDistanceSelect select, SelectCont public SqlFilters convertToSqlFilter(DateDistanceFilter filter, FilterContext filterContext) { Column column = filter.getColumn().resolve(); - String alias = filterContext.getNameGenerator().selectName(filter); - ConnectorSqlTables tables = filterContext.getTables(); + Field dateDistance = createDateDistanceSelect(column , filter.getTimeUnit(), filterContext.getTables(), filterContext.getConversionContext()); - FieldWrapper dateDistanceSelect = createDateDistanceSelect(column, alias, filter.getTimeUnit(), tables, filterContext.getConversionContext()); - ConnectorSqlSelects selects = ConnectorSqlSelects.builder().preprocessingSelect(dateDistanceSelect).build(); - - String eventFilterCteName = tables.getPredecessor(ConceptCteStep.EVENT_FILTER); - Field qualifiedDateDistanceSelect = dateDistanceSelect.qualify(eventFilterCteName).select(); - WhereCondition dateDistanceCondition = new DateDistanceCondition(qualifiedDateDistanceSelect, filterContext.getValue()); + WhereCondition dateDistanceCondition = new DateDistanceCondition(dateDistance, filterContext.getValue()); WhereClauses whereClauses = WhereClauses.builder().eventFilter(dateDistanceCondition).build(); - return new SqlFilters(selects, whereClauses); + return new SqlFilters(ConnectorSqlSelects.none(), whereClauses); } @Override public Condition convertForTableExport(DateDistanceFilter filter, FilterContext filterContext) { Column column = filter.getColumn().resolve(); - String tableName = column.getTable().getName(); - String columnName = column.getName(); - - Field startDateField = DSL.field(DSL.name(tableName, columnName), Date.class); - Field endDate = getEndDate(filterContext.getConversionContext()); + Field dateDistance = createDateDistanceSelect(column , filter.getTimeUnit(), filterContext.getTables(), filterContext.getConversionContext()); - Field dateDistance = filterContext.getFunctionProvider().dateDistance(filter.getTimeUnit(), startDateField, endDate); return new DateDistanceCondition(dateDistance, filterContext.getValue()).condition(); } - private FieldWrapper createDateDistanceSelect( + private Field createDateDistanceSelect( Column column, - String alias, ChronoUnit timeUnit, SqlTables tables, ConversionContext conversionContext ) { - Field startDate = field(DSL.name(tables.getRootTable(), column.getName()), Date.class); + Field startDate = field(name(tables.getRootTable(), column.getName()), Date.class); Field endDate = getEndDate(conversionContext); SqlFunctionProvider functionProvider = conversionContext.getFunctionProvider(); - return new FieldWrapper<>(functionProvider.dateDistance(timeUnit, startDate, endDate).as(alias)); + return functionProvider.dateDistance(timeUnit, startDate, endDate); } private Field getEndDate(ConversionContext conversionContext) { @@ -105,7 +95,7 @@ private Field getEndDate(ConversionContext conversionContext) { ColumnDateRange stratificationDate = conversionContext.getStratificationTable().getQualifiedSelects().getStratificationDate().get(); ColumnDateRange dualColumn = functionProvider.toDualColumn(stratificationDate); // end date is allways treated exclusive, so we get the actual end date when subtracting 1 day - return functionProvider.addDays(dualColumn.getEnd(), DSL.inline(-1)); + return functionProvider.addDays(dualColumn.getEnd(), inline(-1)); } LocalDate endDate; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java index 47f963ac42..13a3d421ab 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java @@ -155,29 +155,19 @@ public ConnectorSqlSelects connectorSelect(FlagSelect flagSelect, SelectContext< } @Override - public SqlFilters convertToSqlFilter(FlagFilter flagFilter, FilterContext> filterContext) { - SqlTables connectorTables = filterContext.getTables(); - String rootTable = connectorTables.getPredecessor(ConceptCteStep.PREPROCESSING); + public SqlFilters convertToSqlFilter(FlagFilter filter, FilterContext> filterContext) { - List> rootSelects = getRequiredColumns(flagFilter.getFlags(), filterContext.getValue()) + List> flagFields = getRequiredColumns(filter.getFlags(), filterContext.getValue()) .stream() - .map(Column::getName) - .map(columnName -> new ExtractingSqlSelect<>(rootTable, columnName, Boolean.class)) - .collect(Collectors.toList()); - - ConnectorSqlSelects selects = ConnectorSqlSelects.builder() - .preprocessingSelects(rootSelects) - .build(); + .map(column -> field(name(column.getTable().getName(), column.getName()), Boolean.class)) + .toList(); - List> flagFields = rootSelects.stream() - .map(sqlSelect -> sqlSelect.qualify(connectorTables.getPredecessor(ConceptCteStep.EVENT_FILTER)).select()) - .toList(); FlagCondition flagCondition = new FlagCondition(flagFields); WhereClauses whereClauses = WhereClauses.builder() .eventFilter(flagCondition) .build(); - return new SqlFilters(selects, whereClauses); + return new SqlFilters(ConnectorSqlSelects.none(), whereClauses); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java index 2b4644f750..762d6da3f8 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java @@ -165,7 +165,7 @@ private CommonAggregationSelect createSumAggregationSelect(Column su ExtractingSqlSelect rootSelect = new ExtractingSqlSelect<>(tables.getRootTable(), sumColumn.getName(), numberClass); preprocessingSelects.add(rootSelect); - String eventFilterCte = tables.cteName(ConceptCteStep.EVENT_FILTER); + String eventFilterCte = tables.cteName(ConceptCteStep.PREPROCESSING); Field sumField = rootSelect.qualify(eventFilterCte).select(); FieldWrapper sumGroupBy; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/AbstractSelectFilterConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/AbstractSelectFilterConverter.java index 65e3ac4980..eb3fbcb898 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/AbstractSelectFilterConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/AbstractSelectFilterConverter.java @@ -10,27 +10,22 @@ import org.jooq.Field; import org.jooq.impl.DSL; +import static com.codahale.metrics.MetricRegistry.name; +import static org.jooq.impl.DSL.field; + abstract class AbstractSelectFilterConverter, T> implements FilterConverter { @Override public SqlFilters convertToSqlFilter(F filter, FilterContext filterContext) { - ExtractingSqlSelect rootSelect = new ExtractingSqlSelect<>( - filterContext.getTables().getPredecessor(ConceptCteStep.PREPROCESSING), - filter.getColumn().getColumn(), - String.class - ); - WhereCondition condition = new MultiSelectCondition( - rootSelect.qualify(filterContext.getTables().getPredecessor(ConceptCteStep.EVENT_FILTER)).select(), + field(name(filterContext.getTables().getRootTable(), filter.getColumn().getColumn()), String.class), getValues(filterContext), filterContext.getFunctionProvider() ); return new SqlFilters( - ConnectorSqlSelects.builder() - .preprocessingSelect(rootSelect) - .build(), + ConnectorSqlSelects.none(), WhereClauses.builder() .eventFilter(condition) .build() @@ -42,7 +37,7 @@ public Condition convertForTableExport(F filter, FilterContext filterContext) Column column = filter.getColumn().resolve(); String tableName = column.getTable().getName(); String columnName = column.getName(); - Field field = DSL.field(DSL.name(tableName, columnName), String.class); + Field field = field(DSL.name(tableName, columnName), String.class); return new MultiSelectCondition(field, getValues(filterContext), filterContext.getFunctionProvider()).condition(); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/MultiSelectCondition.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/MultiSelectCondition.java index b77d7cf0fb..b2de1a04b9 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/MultiSelectCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/MultiSelectCondition.java @@ -1,7 +1,5 @@ package com.bakdata.conquery.sql.conversion.model.filter; -import java.util.Arrays; - import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.google.common.base.Strings; import lombok.RequiredArgsConstructor; @@ -9,6 +7,11 @@ import org.jooq.Field; import org.jooq.impl.DSL; +import java.util.Arrays; + +import static org.jooq.impl.DSL.*; +import static org.jooq.impl.DSL.field; + @RequiredArgsConstructor public class MultiSelectCondition implements WhereCondition { @@ -21,7 +24,7 @@ public WhereCondition negate() { // we want all entries that don't satisfy a condition - because in SQL a comparison with NULL equals UNKNOWN and not FALSE, // we need to check if the entry is NULL or does not fulfil the condition Condition valueIsNull = column.isNull(); - Condition notOrNull = DSL.not(condition()).or(valueIsNull); + Condition notOrNull = not(condition()).or(valueIsNull); return ConditionUtil.wrap(notOrNull); } @@ -35,7 +38,7 @@ public Condition condition() { Condition inCondition = this.functionProvider.in(column, valuesWithoutNull); if (valuesWithoutNull.length < values.length) { - return inCondition.or(DSL.field(column).isNull()); + return inCondition.or(field(column).isNull()); } return inCondition; } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberFilterConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberFilterConverter.java index 051e6d4ff9..7f6d1a8364 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberFilterConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberFilterConverter.java @@ -15,6 +15,9 @@ import org.jooq.Field; import org.jooq.impl.DSL; +import static com.codahale.metrics.MetricRegistry.name; +import static org.jooq.impl.DSL.field; + public class NumberFilterConverter> implements FilterConverter, RANGE> { @Override @@ -24,17 +27,15 @@ public SqlFilters convertToSqlFilter(NumberFilter filter, FilterContext numberClass = NumberMapUtil.getType(column); - ExtractingSqlSelect rootSelect = new ExtractingSqlSelect<>(tables.getRootTable(), column.getName(), numberClass); - Field eventFilterCtePredecessor = rootSelect.qualify(tables.getPredecessor(ConceptCteStep.EVENT_FILTER)).select(); + Field eventFilterCtePredecessor = field(name(tables.getRootTable(), column.getName()), numberClass); + IRange filterValue = filterContext.getValue(); NumberCondition condition = new NumberCondition(eventFilterCtePredecessor, filterValue); - ConnectorSqlSelects selects = ConnectorSqlSelects.builder().preprocessingSelects(List.of(rootSelect)).build(); - WhereClauses whereClauses = WhereClauses.builder().eventFilter(condition).build(); - return new SqlFilters(selects, whereClauses); + return new SqlFilters(ConnectorSqlSelects.none(), whereClauses); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java index 96c74631c6..a490691dca 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java @@ -44,7 +44,7 @@ public ConnectorSqlSelects connectorSelect(ConceptColumnSelect select, SelectCon return ConnectorSqlSelects.none(); } ExtractingSqlSelect connectorColumn = new ExtractingSqlSelect<>(connector.resolveTableId().getTable(), connector.getColumn().getColumn(), Object.class); - ExtractingSqlSelect qualified = connectorColumn.qualify(selectContext.getTables().getPredecessor(ConceptCteStep.EVENT_FILTER)); + ExtractingSqlSelect qualified = connectorColumn.qualify(selectContext.getTables().getPredecessor(ConceptCteStep.PREPROCESSING)); return ConnectorSqlSelects.builder() .preprocessingSelect(connectorColumn) .connectorColumn(Optional.of(qualified)) @@ -119,7 +119,7 @@ private static QueryStep createConnectorColumnSelectQuery( .stream() .filter(tables -> Objects.equals(tables.getRootTable(), connector.resolveTableId().getTable())) .findFirst() - .map(tables -> tables.cteName(ConceptCteStep.EVENT_FILTER)) + .map(tables -> tables.cteName(ConceptCteStep.PREPROCESSING)) .orElse(connector.resolveTableId().getTable()); Table connectorTable = DSL.table(DSL.name(tableName)); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java index ed09464c46..10e332611f 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java @@ -142,7 +142,7 @@ private static Condition containsInfinityDate(ColumnDateRange validityDate, SqlF private static SqlTables createTables(String alias, ConnectorSqlTables connectorTables, Context context) { Map predecessorMapping = new HashMap<>(); - String eventFilterCteName = connectorTables.cteName(EVENT_FILTER); + String eventFilterCteName = connectorTables.cteName(PREPROCESSING); predecessorMapping.putAll(IntervalPackingCteStep.getMappings(context.getDialectBundle())); if (context.getDialectBundle().supportsSingleColumnRanges()) { predecessorMapping.put(UNNEST_DATE, INTERVAL_COMPLETE); @@ -162,7 +162,7 @@ private static QueryStep applyIntervalPacking( ConnectorSqlTables connectorSqlTables, DialectBundle sqlDialect ) { - String eventFilterCteName = connectorSqlTables.cteName(EVENT_FILTER); + String eventFilterCteName = connectorSqlTables.cteName(PREPROCESSING); IntervalPackingContext intervalPackingContext = IntervalPackingContext.builder() .ids(idColumns.qualify(eventFilterCteName)) .daterange(daterange.qualify(eventFilterCteName)) diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java index 0526f5126b..9665f2ad18 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java @@ -119,7 +119,7 @@ private static QueryStep createDistinctSelectCte( SelectContext selectContext ) { // values to aggregate must be event-filtered first - String eventFilterTable = selectContext.getTables().cteName(ConceptCteStep.EVENT_FILTER); + String eventFilterTable = selectContext.getTables().cteName(ConceptCteStep.PREPROCESSING); SingleColumnSqlSelect qualified = preprocessingSelect.qualify(eventFilterTable); SqlIdColumns ids = selectContext.getIds().qualify(eventFilterTable); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java index 422527f8b1..e37d70b2a8 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java @@ -132,7 +132,7 @@ private static FieldWrapper rowNumberField( @RequiredArgsConstructor @Getter enum ValueSelectCteStep implements CteStep { - ROW_NUMBER_STEP("value_select_assign_row_number_step", ConceptCteStep.EVENT_FILTER), ROW_SELECT_STEP("value_select_first_row_step", ROW_NUMBER_STEP); + ROW_NUMBER_STEP("value_select_assign_row_number_step", ConceptCteStep.PREPROCESSING), ROW_SELECT_STEP("value_select_first_row_step", ROW_NUMBER_STEP); private final String suffix; private final CteStep predecessor;