diff --git a/CLAUDE.md b/CLAUDE.md index 3365fabd..af07114f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -37,7 +37,6 @@ norm/ ├── e2e-tests/ # End-to-end tests (standalone, no framework) ├── e2e-tests-micronaut/ # Micronaut integration tests ├── e2e-tests-spring/ # Spring integration tests -├── proto/ # Protocol buffer definitions (internal Wire types) ├── test-scenarios/ # Test scenarios with golden files ├── test-scenarios-frameworks/ # Framework-specific test scenarios └── buildSrc/ # Shared Gradle build logic @@ -52,7 +51,6 @@ norm/ - **Kotlin** - Primary language - **KotlinPoet** - Code generation library (in `generator`) -- **Wire** - Protocol buffer types used as internal model (from `proto/codegen.proto`) - **Testcontainers** - Starts PostgreSQL for JDBC-based schema/query analysis - **Gradle** - Build system with convention plugins in `buildSrc` @@ -84,7 +82,7 @@ Example Gradle tasks: 1. **gradle-plugin** starts a PostgreSQL Testcontainer and applies schema SQL files 2. **JdbcAnalyzer** uses JDBC metadata APIs to build a `Catalog` (tables, columns, enums, domains) and analyze queries (parameter types, result column types) -3. **generator** takes the `Catalog` + analyzed `Query` objects (Wire proto types from `proto/codegen.proto`) and produces Kotlin via KotlinPoet +3. **generator** takes the `Catalog` + analyzed `Query` objects (model data classes in `generator/src/main/kotlin/norm/generator/Model.kt`) and produces Kotlin via KotlinPoet 4. **gradle-plugin** writes the generated `.kt` files ### Runtime Library @@ -136,7 +134,7 @@ Commands: `:one` (single result), `:many` (multiple results), `:execrows` (retur - `gradle-plugin/src/main/kotlin/norm/gradle/NormGenerateTask.kt` - Gradle task orchestrating the pipeline - `runtime/src/main/kotlin/norm/NormDriver.kt` - Core runtime driver - `runtime/src/main/kotlin/norm/Query.kt` - Dynamic query API -- `proto/codegen.proto` - Wire proto definitions for internal model types +- `generator/src/main/kotlin/norm/generator/Model.kt` - Model data classes for internal model types ## Testing diff --git a/generator/src/main/kotlin/norm/generator/ColumnNullabilityAnalyzer.kt b/generator/src/main/kotlin/norm/generator/ColumnNullabilityAnalyzer.kt index f71f4021..d4ed6262 100644 --- a/generator/src/main/kotlin/norm/generator/ColumnNullabilityAnalyzer.kt +++ b/generator/src/main/kotlin/norm/generator/ColumnNullabilityAnalyzer.kt @@ -1,27 +1,18 @@ package norm.generator import org.intellij.lang.annotations.Language +import java.sql.Connection import java.sql.SQLException import java.util.UUID /** * Recursion budget for [ColumnNullabilityAnalyzer.subLinkSubqueryColumnNotNull]: how many levels * of NESTED `ANY_SUBLINK`/`ALL_SUBLINK` (a `SubLink` whose own subselect contains another - * `SubLink`) are resolved before defaulting to nullable — see that method's KDoc. + * `SubLink`) are resolved before defaulting to nullable. * - * This does not prevent an infinite loop: [PgNodeExpression.SubLink.subselectBlock] is - * always extracted, via [PgNodeTreeParser.extractFieldExpression], as a genuine substring of its - * enclosing `SubLink`'s own text, strictly shorter than it — [PgNodeTreeParser] has no mechanism to - * produce a cyclic or self-referential node-tree text, so this recursion is provably bounded by the - * original query text's finite length regardless of this constant's value, or even its presence. - * The budget exists instead as a defensive bound on stack depth and repeated analysis work for a - * pathologically deep (if syntactically legal) chain of nested `= ANY (...)`/`ALL (...)` sublinks, - * the same role [NodeTreeNullabilityAnalyzer.MAX_EXPRESSION_DEPTH] plays for a deeply nested parsed - * expression tree elsewhere in this codebase. Deliberately small: this analysis is only ever needed - * for the (typically shallow) nullability proof of an `IN`/`= ANY` subquery's single output column, - * not for arbitrarily deep query nesting in general — see `QueryAnalysisTest`'s four-level-nesting - * test for a query shape that is semantically not null end-to-end but is reported nullable at this - * budget, pinning that the budget's specific value, not merely its presence, is what is enforced. + * Deliberately small: a chain of four nested `= ANY (...)` sublinks that is semantically NOT NULL + * end-to-end is reported nullable at this budget, pinning that the specific value `3`, not merely + * its presence, is what is enforced. */ private const val SUBLINK_ANALYSIS_DEPTH_BUDGET = 3 @@ -29,21 +20,15 @@ private const val SUBLINK_ANALYSIS_DEPTH_BUDGET = 3 * Recursion budget for [ColumnNullabilityAnalyzer.resolveViewColumnNullability]: how many levels of * nested view-over-view resolution are followed before returning the conservative (nullable) answer. * - * Unlike [SUBLINK_ANALYSIS_DEPTH_BUDGET], this bound is required to prevent a - * `java.lang.StackOverflowError`, not merely to bound work: resolution recurses through real JVM - * stack frames, and `CREATE VIEW` permits unbounded nesting. `50` is far past any real schema's - * nesting depth and fits in a 512 KiB thread stack, so a small Gradle worker thread is safe. Actual + * Required to prevent a `java.lang.StackOverflowError`: resolution recurses through real JVM stack + * frames, and `CREATE VIEW` permits unbounded nesting. `50` fits in a 512 KiB thread stack. Actual * cycles, possible at much shallower depths, are handled by * [ColumnNullabilityAnalyzer.viewColumnNullabilityInProgress] instead. * - * Views within this many levels of the deepest point of a chain that itself exceeds the budget can - * get either the truncated or the fully-resolved answer, depending on which views were memoized - * first: the depth check runs before the memo lookup, so a relid at or past the budget always - * truncates, but an untainted cached answer for a strictly shallower relid is reused without - * revisiting this guard. Both answers are sound — truncation only widens — and the sweep order is - * fixed (see `PgCatalogLoader.loadViewColumnNamesByRelidAndAttnum`), so a given schema always - * generates the same Kotlin. Closing the residual would need per-entry tracking of how much depth - * each cached answer needed, which this guard's narrow job does not warrant. + * A relid within this many levels of the deepest point of a chain that itself exceeds the budget + * can get either the truncated or the fully-resolved answer, depending on which views were memoized + * first, since the depth check runs before the memo lookup. Both answers are sound — truncation + * only widens — and the sweep order is fixed, so a given schema always generates the same Kotlin. */ internal const val VIEW_NULLABILITY_RECURSION_DEPTH_BUDGET = 50 @@ -51,18 +36,14 @@ internal const val VIEW_NULLABILITY_RECURSION_DEPTH_BUDGET = 50 * One result column's [nullable] flag (`true` means nullable) together with, when resolvable, its * [provenanceExpression] — the CTE-body SQL expression, verbatim from the developer's own query * text, for a column whose select item is merely a bare reference into a CTE's output. `null` when - * [NodeTreeProvenanceResolver] found no CTE reference to attribute this column to, or - * [resolveNodeTreeProvenanceExpression] could not prove the expression correct. + * no CTE reference could be attributed to this column, or the expression could not be proven + * correct. * * Also carries [originalColumnName] — the real source column name, resolved from the outer target - * entry's own `:resorigtbl`/`:resorigcol` (see [PgCatalogLoader.columnNameByRelidAndAttnum]) rather - * than whatever alias the select item's text happens to spell. `null` when those fields are `0` (no - * single source column) or the OID/attnum pair isn't in the catalog map — the caller must fall back - * to its ordinary column-name resolution, never guess. - * - * Carries all three facts together, rather than as separate parallel lists, because - * [ColumnNullabilityAnalyzer.queryColumnNullabilityViaProsqlbody] resolves them from the same parsed - * node tree in one round trip. + * entry's own `:resorigtbl`/`:resorigcol` rather than whatever alias the select item's text happens + * to spell. `null` when those fields are `0` (no single source column) or the OID/attnum pair isn't + * in the catalog map — the caller must fall back to its ordinary column-name resolution, never + * guess. */ internal data class ColumnAnalysis( val nullable: Boolean, @@ -71,38 +52,108 @@ internal data class ColumnAnalysis( ) /** - * Drives per-column nullability analysis for a SQL query on behalf of [loader]: fetching the - * query's own parsed node tree (via `prosqlbody` or a probe function, see - * [queryColumnNullabilityViaProsqlbody]'s own KDoc), then recursively resolving CTE bodies, - * subqueries, and `MERGE` actions to feed [NodeTreeNullabilityAnalyzer] the source-column - * not-null information it needs to evaluate each result column's expression. + * Returns `true` when `(varno, varattno)` is proven non-null by the query's `WHERE` clause, + * either directly or through the GROUP RTE remap (target-list `Var`s point at the GROUP RTE + * when `hasGroupRTE`, while `WHERE`-clause `Var`s use base relation varnos). + */ +private fun isProvenByQuals( + qualNotNullVars: Set>, + groupRteMap: Map, Pair>, + varno: Int, + varattno: Int, +): Boolean = qualNotNullVars.contains(varno to varattno) || + groupRteMap[varno to varattno]?.let { qualNotNullVars.contains(it) } == true + +/** + * Everything needed to answer `isSourceColumnNotNull` — how a `Var` reference inside one query + * block resolves to a source column's not-null answer — for a single query block, whether that + * block is [ColumnNullabilityAnalyzer]'s own outermost statement, a CTE body, a `FROM`-clause + * subquery, or a `SubLink`'s subselect. Built by [ColumnNullabilityAnalyzer.buildQueryBlockScope] + * so all four call shapes share exactly one fallback chain. * - * Split out from [loader]'s own catalog-loading responsibilities (schema introspection, function - * metadata, safe-list lookups) because this is a distinct concern: [loader] answers "what does the - * catalog say", while this class answers "is this specific query's result column nullable" by - * combining catalog answers with the query's own parsed structure. + * [qualProvenVars] and [groupRteMap] are computed empty, so their branches below never fire, when + * the query block has GROUPING SETS/CUBE/ROLLUP (which null-extends a grouping key AFTER `WHERE` + * has already filtered rows) or is itself an `INSERT`/`UPDATE`/`DELETE`/`MERGE` (whose own `WHERE` + * clause can test a column value its `SET` clause, or a `MERGE` action, is about to overwrite). + * + * @property rangeTable varno to relid, base tables only. + * @property hasGroupingSets `true` when the query block uses `GROUPING SETS`, `CUBE`, or `ROLLUP`. + * @property groupRteMap `(groupVarno, attrPos)` to `(baseVarno, baseVarattno)`, empty whenever + * [hasGroupingSets]. + * @property qualProvenVars `(varno, varattno)` pairs the query block's own `WHERE` clause proves + * non-null, empty whenever qual narrowing does not apply. + * @property ownCtes CTE bodies declared directly in the query block's own `:cteList`, keyed by + * name. + * @property enclosingCtes CTE bodies visible via `:ctelevelsup` greater than `0` — declared in + * whichever scope encloses the query block, never its own nested `WITH` clause. Empty for the + * outermost statement, which has no enclosing scope to point past. + * @property cteReferences varno to CTE reference, for a `Var` whose range-table entry is a CTE + * rather than a base table or subquery. + * @property subqueryColumnNotNull `(varno, varattno)` to `true` for a `FROM`-clause subquery RTE + * column already proven non-null by recursively analyzing that subquery's own target list. + * @property mergeAbsentVarnos varno to whether that relation can be entirely absent for some + * result row, only when the query block is itself a `MERGE` — empty for every other shape. + * @property forceNewNullable `true` when a `RETURNING WITH (OLD AS o, NEW AS n)` reference to + * `NEW` must be forced nullable. + * @property resultRelationVarno the query block's own `:resultRelation` varno, `0` for a plain + * `SELECT`. + */ +private class QueryBlockScope( + val rangeTable: Map, + val hasGroupingSets: Boolean, + val groupRteMap: Map, Pair>, + val qualProvenVars: Set>, + val ownCtes: Map>, + val enclosingCtes: Map>, + val cteReferences: Map, + val subqueryColumnNotNull: Map, Boolean>, + val mergeAbsentVarnos: Map, + val forceNewNullable: Boolean, + val resultRelationVarno: Int, +) { + /** + * The source-column-resolution chain every query block shape resolves a `Var` through: a + * `MERGE` relation `EXPLAIN` proved can be entirely absent for some result row, a `WHERE`-clause + * qual (directly or through the GROUP RTE remap), a base-table relation's own catalog constraint + * (via [isColumnNotNull]), a GROUP RTE remapped back to its base column, a `FROM`-clause + * subquery's already-resolved column, or a CTE reference resolved against whichever of + * [ownCtes]/[enclosingCtes] its own `:ctelevelsup` selects. + */ + fun isSourceColumnNotNull(varno: Int, varattno: Int, isColumnNotNull: (Pair) -> Boolean): Boolean { + if (mergeAbsentVarnos[varno] == true) return false + if (isProvenByQuals(qualProvenVars, groupRteMap, varno, varattno)) return true + rangeTable[varno]?.let { relid -> return isColumnNotNull(relid to varattno) } + groupRteMap[varno to varattno]?.let { (baseVarno, baseAttno) -> + val baseRelid = rangeTable[baseVarno] ?: return false + return isColumnNotNull(baseRelid to baseAttno) + } + if (subqueryColumnNotNull[varno to varattno] == true) return true + val reference = cteReferences[varno] ?: return false + val ctesInScope = if (reference.ctelevelsup == 0) ownCtes else enclosingCtes + return ctesInScope[reference.name]?.getOrNull(varattno - 1) == false + } +} + +/** + * Drives per-column nullability analysis for a SQL query: fetching the query's own parsed node + * tree via a temporary `prosqlbody` probe function, then recursively resolving CTE bodies, + * subqueries, and `MERGE` actions to feed + * [NodeTreeNullabilityAnalyzer] the source-column not-null information it needs to evaluate each + * result column's expression. [catalog] answers "what does the catalog say" (function strictness, + * safe-list membership, column not-null facts); this class combines those answers with the + * query's own parsed structure to answer "is this specific result column nullable". */ -internal class ColumnNullabilityAnalyzer(private val loader: PgCatalogLoader) { - private val connection get() = loader.connection - private val nodeTreeParser get() = loader.nodeTreeParser - private val columnNotNullByRelidAndAttnum get() = loader.columnNotNullByRelidAndAttnum - private val columnNameByRelidAndAttnum get() = loader.columnNameByRelidAndAttnum - private val aggregateHasNonNullInitialValue get() = loader.aggregateHasNonNullInitialValue - private val alwaysNonNullFunctionOids get() = loader.alwaysNonNullFunctionOids - private val neverNullForNonNullInputOids get() = loader.neverNullForNonNullInputOids - private val lagLeadWithDefaultOids get() = loader.lagLeadWithDefaultOids - private val immutableFunctionOids get() = loader.immutableFunctionOids - private val nonNullIffFirstArgumentNonNullFunctionOids get() = loader.nonNullIffFirstArgumentNonNullFunctionOids - private val isStrictFunction get() = loader.isStrictFunction +internal class ColumnNullabilityAnalyzer(private val connection: Connection, private val catalog: NullabilityCatalog) { + private val nodeTreeParser = PgNodeTreeParser() /** * Memoized per-relid view-column nullability, populated by [resolveViewColumnNullability]. Index - * `i` (0-based) corresponds to attnum `i + 1`. A `null` VALUE, as opposed to an absent key, means the - * relid is not a view or materialized view at all, so [isColumnNotNull] must fall through to + * `i` (0-based) corresponds to attnum `i + 1`. A `null` VALUE, as opposed to an absent key, means + * the relid is not a view or materialized view at all, so [isColumnNotNull] must fall through to * base-table resolution. * * Written unconditionally by every successful resolution, even for a tainted answer — see - * [viewColumnNullabilityTaintedRelids]. Needs no synchronization (single-threaded [connection]). + * [viewColumnNullabilityTaintedRelids]. */ private val viewColumnNullabilityMemo = mutableMapOf?>() @@ -110,11 +161,10 @@ internal class ColumnNullabilityAnalyzer(private val loader: PgCatalogLoader) { * Relids currently being resolved by [resolveViewColumnNullability] — guards against infinite * recursion through a view dependency cycle. * - * Not merely defensive: `CREATE OR REPLACE VIEW` only requires the relations the new - * definition references to exist at replace time, so both a mutual cycle (`a` selects from `b`, - * then `b` is replaced to select from `a`) and a direct self-cycle are constructible. PostgreSQL - * refuses to query such a view at all, so the nullable placeholder this guard returns has no true - * answer to under-approximate. + * Not merely defensive: `CREATE OR REPLACE VIEW` only requires the relations the new definition + * references to exist at replace time, so both a mutual cycle (`a` selects from `b`, then `b` is + * replaced to select from `a`) and a direct self-cycle are constructible, even though PostgreSQL + * refuses to query such a view at all. */ private val viewColumnNullabilityInProgress = mutableSetOf() @@ -127,8 +177,8 @@ internal class ColumnNullabilityAnalyzer(private val loader: PgCatalogLoader) { /** * Total number of taint events since construction — monotonically increasing, never reset; - * [resolveViewColumnNullability] compares entry and exit snapshots to decide whether its own answer - * was built from one. See [viewColumnNullabilityTaintedRelids] for what counts and why. + * [resolveViewColumnNullability] compares entry and exit snapshots to decide whether its own + * answer was built from one. */ private var viewColumnNullabilityTaintEventCount = 0 @@ -137,25 +187,20 @@ internal class ColumnNullabilityAnalyzer(private val loader: PgCatalogLoader) { * directly, or transitively through another tainted relid's cached answer — rather than from a * genuine, order-independent evaluation of the view's own defining query. * - * A taint event is the cycle guard firing, the depth guard firing, an unanalyzable node tree, or a - * memo READ of an already-tainted relid. Any frame whose own entry-to-exit window spans one built - * its answer from it, so its relid is marked here too. That last case is required, not merely - * convenient: a sibling reaching a tainted relid purely through the memo fast path does no recursion - * of its own and would otherwise see no counter movement and treat itself as untainted. + * A taint event is the cycle guard firing, the depth guard firing, an unanalyzable node tree, or + * a memo READ of an already-tainted relid. A memo-fast-path read must still count: it does no + * recursion of its own, so without this it would see no counter movement and treat itself as + * untainted even when the cached answer it reused was itself tainted. * - * Tainted entries are EVICTED once the outermost call finishes, rather than never cached at all. - * Never caching also removes cross-call order dependence, but makes every reference to a shared - * tainted ancestor re-walk its whole subtree, turning a branching view graph exponential. Evicting - * keeps each relid resolved at most once per top-level call while still recomputing a poisoned - * answer for the next, independent query. + * Tainted entries are EVICTED once the outermost call finishes, rather than never cached at all, + * so each relid is resolved at most once per top-level call while a later, independent query still + * recomputes a poisoned answer. */ private val viewColumnNullabilityTaintedRelids = mutableSetOf() /** - * Replaces `?` parameter placeholders in [sql] with typed non-null sentinel values. - * - * Uses `PreparedStatement.getParameterMetaData()` to determine the PostgreSQL type of each - * parameter, then builds a non-null literal of that type (e.g., `0::int4`, `''::text`). + * Replaces `?` parameter placeholders in [sql] with typed non-null sentinel values (e.g., + * `0::int4`, `''::text`). * * @return The SQL with `?` replaced by typed sentinels, or `null` if parameter metadata * cannot be obtained (caller should fall back to NULL replacement). @@ -169,7 +214,7 @@ internal class ColumnNullabilityAnalyzer(private val loader: PgCatalogLoader) { nonNullSentinel(parameterMetaData.getParameterTypeName(index)) } } - replaceParameterPlaceholdersWithSentinels(sql, sentinels) + replaceParameterPlaceholders(sql) { sentinels.getOrElse(it) { "NULL" } } } catch (_: SQLException) { null } @@ -177,16 +222,14 @@ internal class ColumnNullabilityAnalyzer(private val loader: PgCatalogLoader) { /** * For [nodeTree]'s own outermost statement — never recursing into a CTE it declares; each CTE's - * own body resolves its own MERGE independently (see [analyzeCteBodyNullability]) — determines - * which of its two base-table relations (identified by `:rtable` varno) can be entirely absent - * for some result row, via [explainMergeSideNullability] rather than `:mergeActionList`/text - * inspection. + * own body resolves its own MERGE independently — determines which of its two base-table + * relations (identified by `:rtable` varno) can be entirely absent for some result row, via + * [explainMergeSideNullability]. * * A `MERGE`'s match-optionality (`WHEN NOT MATCHED BY SOURCE`, `WHEN NOT MATCHED [BY TARGET] - * THEN INSERT`) is invisible to `:varnullingrels`: a `MERGE ... WHEN NOT MATCHED - * BY SOURCE THEN DELETE RETURNING src.col` has an empty `:varnullingrels` on `src.col`'s `Var`, - * identical to an ordinary, always-present reference. [explainMergeSideNullability]'s KDoc has - * the full reasoning for why `EXPLAIN`'s own join type answers this precisely instead. + * THEN INSERT`) is invisible to `:varnullingrels`: a `MERGE ... WHEN NOT MATCHED BY SOURCE THEN + * DELETE RETURNING src.col` has an empty `:varnullingrels` on `src.col`'s `Var`, identical to an + * ordinary, always-present reference. * * @param sql the EXACT (already sentinel-substituted) statement text to run `EXPLAIN` against — * the whole top-level statement, including any leading `WITH` clause, so a `MERGE` nested @@ -207,11 +250,10 @@ internal class ColumnNullabilityAnalyzer(private val loader: PgCatalogLoader) { if (nodeTreeParser.parseCommandType(nodeTree) != PgNodeTreeParser.COMMAND_TYPE_MERGE) return emptyMap() val targetVarno = nodeTreeParser.parseResultRelation(nodeTree) // A RETURNING list that only reads the target relation's own columns, or OLD/NEW references, - // never needs EXPLAIN's resolution at all — see containsVarOutsideRelation's KDoc. Skipping it - // here matters beyond saving an EXPLAIN round trip: a MERGE whose USING source is not a plain - // base table or CTE (e.g. a VALUES list or a subquery) can never be resolved below, but that - // must not block a RETURNING list that never depended on knowing which side of that join is - // nullable. + // never needs EXPLAIN's resolution at all. Skipping it here matters beyond saving an EXPLAIN + // round trip: a MERGE whose USING source is not a plain base table or CTE (e.g. a VALUES list + // or a subquery) can never be resolved below, but that must not block a RETURNING list that + // never depended on knowing which side of that join is nullable. val returningEntries = nodeTreeParser.parseReturningList(nodeTree) if (returningEntries.none { NodeTreeNullabilityAnalyzer.containsVarOutsideRelation(it.expression, targetVarno) }) { return emptyMap() @@ -221,9 +263,7 @@ internal class ColumnNullabilityAnalyzer(private val loader: PgCatalogLoader) { // the target — the source, of any rtekind. A `USING` clause with more than one relation of its // own (e.g. a join or subquery source) has no single relation this method can attribute a join // side to, so it bails rather than guess. Reads the FULL range table, not [rangeTable] (base - // tables only) — a CTE source's own varno never appears there at all — since the returned map - // is keyed by varno regardless of the relation's kind, exactly matching how a caller's `Var` - // (base-table or CTE) looks it up later. + // tables only), since a CTE source's own varno never appears there at all. val sourceEntries = nodeTreeParser.parseRangeTableEntries(nodeTree).filterKeys { it != targetVarno } if (sourceEntries.size != 1) return null val (sourceVarno, sourceEntry) = sourceEntries.entries.single() @@ -242,21 +282,11 @@ internal class ColumnNullabilityAnalyzer(private val loader: PgCatalogLoader) { * chooses to execute it. * * A plain base-table source ([RangeTableEntry.Relation]) has exactly one name: its own catalog - * name, via [resolveTableName]. - * - * A CTE source ([RangeTableEntry.Cte]) can appear either way in the plan, and nothing in the - * parsed query tree says which the planner will pick, a cost-based decision made only at plan - * time: - * - `MATERIALIZED`, or otherwise not eligible for inlining, it plans as its own `"CTE Scan"` node - * carrying `"CTE Name"` set to the literal name from the `WITH` clause. - * - referenced only once (always true of a `MERGE ... USING` source) and eligible for inlining, - * PostgreSQL folds it directly into whatever it scans, and the CTE's own name never appears in - * the plan. [resolveInlinedBaseRelationName] recovers a second candidate name for this shape, - * but only for the simplest possible body — nothing but `SELECT ... FROM oneBaseTable` — since - * anything with a join, a second relation, or a derived table has no single name to offer. - * - * Offering both candidates together never risks a false attribution: for a given plan, at most one - * of them can ever actually appear. + * name, via [resolveTableName]. A CTE source ([RangeTableEntry.Cte]) can appear in the plan + * either under its own `WITH`-clause name (a `"CTE Scan"` node, when not inlined) or, when + * PostgreSQL inlines it into whatever it scans, under the name [resolveInlinedBaseRelationName] + * recovers — only for the simplest possible body, `SELECT ... FROM oneBaseTable`. Offering both + * candidates together never risks a false attribution: for a given plan, at most one can appear. * * @return `null` when [sourceEntry] is neither a base table nor a CTE (a join, subquery, function, * `VALUES`, or another `rtekind` this cannot safely name) @@ -273,9 +303,7 @@ internal class ColumnNullabilityAnalyzer(private val loader: PgCatalogLoader) { /** * The bare table name [cteName]'s body resolves to, only when that body is nothing but a plain - * `SELECT ... FROM oneBaseTable` — a single `rtekind 0` range-table entry and nothing else. See - * [mergeSourceRelationNameCandidates]'s own KDoc for why this narrow shape is the only one this - * offers as an inlining candidate. + * `SELECT ... FROM oneBaseTable` — a single `rtekind 0` range-table entry and nothing else. * * @return `null` when [cteName] cannot be found in [nodeTree]'s own `:cteList`, or its body is * anything other than exactly one base-table range-table entry @@ -292,50 +320,40 @@ internal class ColumnNullabilityAnalyzer(private val loader: PgCatalogLoader) { * SQL-standard (`BEGIN ATOMIC ... END`) function body. * * `prosqlbody` holds the same post-parse-analysis `{QUERY ...}` node shape `pg_rewrite.ev_action` - * does (neither is rewriter-expanded, so nested views stay relation RTEs in both), but — unlike - * `CREATE VIEW` — PostgreSQL populates it for `UPDATE`/`DELETE`/`MERGE ... RETURNING` and for - * data-modifying CTEs, because only `CREATE VIEW` itself rejects a data-modifying statement, not - * a SQL-standard function body. [analyzeNodeTree] reads the same field shape either way, since - * [PgNodeTreeParser]'s extraction methods locate each field by a depth-one scan starting at the - * first unescaped `{`, ignoring whatever wrapping parentheses precede or follow it. + * does, but — unlike `CREATE VIEW` — PostgreSQL populates it for `UPDATE`/`DELETE`/`MERGE ... + * RETURNING` and for data-modifying CTEs, because only `CREATE VIEW` itself rejects a + * data-modifying statement, not a SQL-standard function body. * * The function is created with ZERO arguments: a real `$n` parameter would appear as a `PARAM` - * node in the tree, which [NodeTreeNullabilityAnalyzer] has no case for and would fall through to - * its `Unknown` (safe-nullable) handling for every expression that touches it — silently - * widening every parameter-touching column. [sql]'s own `?` placeholders are therefore replaced - * with typed non-null sentinel literals internally, via [buildViewSqlWithSentinels], before this - * method is ever invoked. + * node, silently widening every parameter-touching column to nullable. [sql]'s own `?` + * placeholders are therefore replaced with typed non-null sentinel literals internally, via + * [buildViewSqlWithSentinels], before this method is ever invoked. * * A statement with no result columns at all (an `INSERT`/`UPDATE`/`DELETE`/`MERGE` without * `RETURNING`) fails PostgreSQL's `RETURNS SETOF record` check on function creation — there is * nothing to probe, and the [SQLException] is caught here rather than propagated. * - * Provenance piggybacks the same round trip: [NodeTreeProvenanceResolver] resolves each column's - * CTE-body position from the identical [nodeTree] this method already builds for nullability, and - * [resolveNodeTreeProvenanceExpression] extracts and cross-validates the actual expression text - * from [sql] — the original, un-substituted query text, never [substitutedSql] — so a sentinel - * literal built only to satisfy a `?`'s type can never leak into generated KDoc. + * Provenance piggybacks the same round trip: each column's expression text is extracted and + * cross-validated from [sql] — the original, un-substituted query text, never [substitutedSql] — + * so a sentinel literal built only to satisfy a `?`'s type can never leak into generated KDoc. * * @param sql the SQL query or DML statement to analyze; any `?` parameter placeholder is - * replaced with a typed non-null sentinel literal internally (see [buildViewSqlWithSentinels]) - * only for building the probe function — [sql] itself, `?` intact, is what provenance - * expression text is extracted from + * replaced with a typed non-null sentinel literal internally only for building the probe + * function — [sql] itself, `?` intact, is what provenance expression text is extracted from * @return one [ColumnAnalysis] per result column in `SELECT`/`RETURNING` order, or `null` if * [sql] has no result columns to probe or the probe itself failed for any reason — the caller * must treat `null` as "this path has no answer", never as "zero columns." */ internal fun queryColumnNullabilityViaProsqlbody(@Language("PostgreSQL") sql: String): List? { - val substitutedSql = buildViewSqlWithSentinels(sql) ?: replaceParameterPlaceholders(sql) + val substitutedSql = buildViewSqlWithSentinels(sql) ?: replaceParameterPlaceholders(sql) { "NULL" } val functionName = "norm_nullability_${UUID.randomUUID().toString().replace("-", "")}" return try { connection.createStatement().use { statement -> statement.execute( "CREATE FUNCTION pg_temp.$functionName() RETURNS SETOF record LANGUAGE sql " + - // The newline before "; END" is required, not style: [substitutedSql] is caller- - // supplied SQL text that can legitimately end in a trailing `--` line comment (ordinary - // in a queries.sql), which extends to end of line. Without a newline separating it from - // "; END", the comment swallows the terminator too, and PostgreSQL sees unterminated - // input instead of a syntax error naming the real cause. + // The newline before "; END" is required, not style: substitutedSql can legitimately + // end in a trailing `--` line comment, which extends to end of line; without a newline + // separating it from "; END", the comment swallows the terminator too. "BEGIN ATOMIC $substitutedSql\n; END", ) } @@ -343,9 +361,8 @@ internal class ColumnNullabilityAnalyzer(private val loader: PgCatalogLoader) { val nodeTree = connection.createStatement().use { statement -> statement.executeQuery( "SELECT prosqlbody::text FROM pg_proc " + - // "pg_temp" is a per-session ALIAS, not a literal schema name — the real catalog row - // is named "pg_temp_N", so 'pg_temp'::regnamespace fails to resolve at all (verified - // live: "ERROR: schema "pg_temp" does not exist"). pg_my_temp_schema() returns the + // "pg_temp" is a per-session ALIAS, not a literal schema name: 'pg_temp'::regnamespace + // fails with `ERROR: schema "pg_temp" does not exist`. pg_my_temp_schema() returns the // current session's actual temp schema OID directly. "WHERE proname = '$functionName' AND pronamespace = pg_my_temp_schema()", ).use { resultSet -> @@ -353,14 +370,14 @@ internal class ColumnNullabilityAnalyzer(private val loader: PgCatalogLoader) { resultSet.getString(1) } } - val rangeTable = nodeTreeParser.parseRangeTable(nodeTree) + val rangeTable = nodeTreeParser.parseRangeTableEntries(nodeTree).baseRelations() val mergeAbsent = mergeAbsentVarnos(nodeTree, rangeTable, substitutedSql) ?: return null // '?' in sql, not substitutedSql: a sentinel-substituted CONST is byte-identical to a // hand-written literal once embedded in the SQL text — the parsed tree retains no memory // of which one it was. trustAssignedExpressions=false whenever the original sql had any - // parameter blocks analyzeNodeTree's :targetList-to-:returningList substitution (see its - // KDoc) for the whole statement, not just the specific assignment a parameter feeds, - // because there is no structural way from here to tell which assignment(s) it was. + // parameter blocks the :targetList-to-:returningList substitution for the whole statement, + // not just the specific assignment a parameter feeds, since there is no structural way from + // here to tell which assignment(s) it was. val nullability = analyzeNodeTree( nodeTree, applyQualNarrowing = true, @@ -394,16 +411,15 @@ internal class ColumnNullabilityAnalyzer(private val loader: PgCatalogLoader) { * Each entry's own `:resorigtbl`/`:resorigcol` — [TargetEntry.originalTableOid] and * [TargetEntry.originalColumnNumber] — name the column PostgreSQL itself traced this result * column back to, walking through a CTE or subquery reference rather than stopping at the - * immediate select item's own alias (`WITH c AS (SELECT id AS parent_id FROM parent) SELECT + * immediate select item's own alias: `WITH c AS (SELECT id AS parent_id FROM parent) SELECT * parent_id FROM c`'s outer target entry carries `:resorigtbl`/`:resorigcol` for `parent.id`, not - * the CTE's own `parent_id` alias). + * the CTE's own `parent_id` alias. * * @return one entry per result column: the resolved column name, or `null` when * [TargetEntry.originalTableOid]/[TargetEntry.originalColumnNumber] is `0` (no single source * column — a computed expression, an aggregate, a set-operation branch, or a `USING`/`NATURAL` - * merged join column) or the OID/attnum pair is absent from [columnNameByRelidAndAttnum] for any - * other reason. The caller must treat `null` as "fall back to the ordinary resolution", never - * guess a value. + * merged join column) or the OID/attnum pair is absent from the catalog map for any other + * reason. The caller must treat `null` as "fall back to the ordinary resolution", never guess. */ private fun resolveOriginalColumnNames(nodeTree: String): List { val returningEntries = nodeTreeParser.parseReturningList(nodeTree) @@ -412,7 +428,7 @@ internal class ColumnNullabilityAnalyzer(private val loader: PgCatalogLoader) { if (entry.originalTableOid == 0 || entry.originalColumnNumber == 0) { null } else { - columnNameByRelidAndAttnum[entry.originalTableOid to entry.originalColumnNumber] + catalog.columnNameByRelidAndAttnum[entry.originalTableOid to entry.originalColumnNumber] } } } @@ -420,22 +436,15 @@ internal class ColumnNullabilityAnalyzer(private val loader: PgCatalogLoader) { /** * Computes per-column nullability from [nodeTree] — the `pg_rewrite.ev_action` text of a * temporary view, or the `pg_proc.prosqlbody` text of a temporary probe function; both share the - * same post-parse-analysis `{QUERY ...}` node shape (see [queryColumnNullabilityViaProsqlbody]'s - * KDoc). + * same post-parse-analysis `{QUERY ...}` node shape. * - * Reads the `:targetList` first — this covers every plain `SELECT`, including one that reaches - * this function only because it CONTAINS a data-modifying CTE (the outer statement is still a - * `SELECT`, so PostgreSQL's own `CREATE VIEW` restriction, and by extension nothing here, ever - * blocked it). Falls back to `:returningList` when the outer statement's own target list is - * empty — true only for a topmost `UPDATE`/`DELETE`/`MERGE ... RETURNING`: a plain `SELECT` - * always has a non-empty target list, or [connection] would have rejected it as a query with no - * result columns before ever reaching this point. + * Reads `:returningList` when non-empty (a topmost `UPDATE`/`DELETE`/`MERGE ... RETURNING`), + * otherwise `:targetList` (every plain `SELECT`, including one that reaches this function only + * because it CONTAINS a data-modifying CTE). * * @param applyQualNarrowing When `false`, disables `WHERE`-clause qual narrowing * ([NodeTreeNullabilityAnalyzer.qualProvenNonNullVars]) for this entire call — the top-level - * query AND every nested CTE body and subquery reached from it. Threaded through rather than a - * fixed `true` so a future caller with a rewritten body whose `WHERE`/`ON` predicate no longer - * corresponds to what `RETURNING` sees can suppress it; every current caller passes `true`. + * query AND every nested CTE body and subquery reached from it. */ private fun analyzeNodeTree( nodeTree: String, @@ -444,126 +453,44 @@ internal class ColumnNullabilityAnalyzer(private val loader: PgCatalogLoader) { trustAssignedExpressions: Boolean = true, mergeAbsentVarnos: Map = emptyMap(), ): List { - val rangeTable = nodeTreeParser.parseRangeTable(nodeTree) // varno → relid (base tables only) - // GROUP BY queries use an *GROUP* RTE (rtekind 9) whose target list VARs reference the group - // entry varno instead of the base table varno directly. Resolve these back to their base table - // column so isSourceColumnNotNull can check pg_attribute.attnotnull correctly. - // - // EXCEPTION: When GROUPING SETS, CUBE, or ROLLUP is used, GROUP BY columns can receive NULL - // for rows where the column is not part of the current grouping set. PostgreSQL 18 enforces - // this via a *GROUP* RTE (rtekind 9): target list VARs reference the GROUP RTE instead of the - // base table, so parseRangeTable() can't find them and they fall back to nullable on their - // own. PostgreSQL 16/17 have no GROUP RTE, so this exception skips GROUP RTE resolution - // entirely; the actual nullability override for grouping keys (including EXPRESSION keys - // such as `ROLLUP(lower(a))`, which never produce a bare {VAR } target-list entry to remap - // here) is applied by NodeTreeNullabilityAnalyzer.extractColumnNullability via the - // hasGroupingSets flag passed to buildAnalyzer below. - val hasGroupingSets = nodeTreeParser.hasGroupingSets(nodeTree) - val groupRteMap = if (hasGroupingSets) { - emptyMap() - } else { - nodeTreeParser.parseGroupRteMap(nodeTree) // (groupVarno, attrPos) → (baseVarno, baseVarattno) - } - // Computed once so the same resolution feeds both buildCteColumnNotNull's varno-keyed projection - // and buildAnalyzer's resolvedCtes, which needs the raw, name-keyed map. - val resolvedCtes = resolveCteBodies(nodeTree, applyQualNarrowing, sql) - // For subquery RTEs (rtekind 1), the outer VAR's varno is not in rangeTable. - // Resolve their nullability by recursively analyzing each subquery's target list. - // The map is keyed by (varno, varattno) for direct lookup in isSourceColumnNotNull. - val subqueryColumnNotNull = buildSubqueryColumnNotNull(nodeTree, resolvedCtes, applyQualNarrowing, sql) - val cteColumnNotNull = buildCteColumnNotNull(nodeTree, resolvedCtes) - // A non-zero :resultRelation means this is an INSERT/UPDATE/DELETE/MERGE, not a SELECT — see - // parseResultRelation's KDoc. Its :targetList holds the value expressions being written to - // each explicitly-assigned column of the target relation (keyed by :resno = the column's - // attribute number), which is exactly what a :returningList Var referencing that same - // (resultRelationVarno, attno) pair actually reads back — not the column's general catalog - // constraint, which says nothing about what this statement is about to write. See - // targetListByResno's use below. - val resultRelationVarno = nodeTreeParser.parseResultRelation(nodeTree) - val targetListByResno = if (resultRelationVarno == 0 || !trustAssignedExpressions) { - // !trustAssignedExpressions means the original sql (before sentinel substitution) contained - // a `?` parameter placeholder somewhere — see queryColumnNullabilityViaProsqlbody's call - // site KDoc. A sentinel-substituted CONST is byte-identical, in the parsed tree, to a - // hand-written literal: there is no structural signal left to tell "the caller supplied - // this at runtime, and could supply NULL" from "the query text itself guarantees this value" - // for any specific assignment, so trusting :targetList at all is unsafe for the whole - // statement once any parameter exists anywhere in it. `INSERT INTO t(name) - // VALUES (?) RETURNING name` reports NOT NULL if the sentinel substitution is trusted here, - // even though the caller can bind an actual `NULL` for that exact parameter. + val scope = + buildQueryBlockScope(nodeTree, emptyMap(), applyQualNarrowing, sql, mergeAbsentVarnos = mergeAbsentVarnos) + // A non-zero :resultRelation means this is an INSERT/UPDATE/DELETE/MERGE, not a SELECT. Its + // :targetList holds the value expressions being written to each explicitly-assigned column of + // the target relation (keyed by :resno = the column's attribute number), which is exactly what + // a :returningList Var referencing that same (resultRelationVarno, attno) pair actually reads + // back — not the column's general catalog constraint, which says nothing about what this + // statement is about to write. + val targetListByResno = if (scope.resultRelationVarno == 0 || !trustAssignedExpressions) { + // !trustAssignedExpressions means the original sql (before sentinel substitution) contained a + // `?` parameter placeholder somewhere. A sentinel-substituted CONST is byte-identical, in the + // parsed tree, to a hand-written literal: there is no structural signal left to tell "the + // caller supplied this at runtime, and could supply NULL" from "the query text itself + // guarantees this value", so trusting :targetList at all is unsafe once any parameter exists + // anywhere in the statement. `INSERT INTO t(name) VALUES (?) RETURNING name` reports NOT NULL + // if the sentinel substitution is trusted here, even though the caller can bind `NULL`. emptyMap() } else { // rangeTable[resultRelationVarno] is only present for an ordinary base-table target (rtekind - // 0) — never null for a real INSERT/UPDATE/DELETE/MERGE, since PostgreSQL requires a real - // relation to write to, but defensively treated as "substitution unsafe" (empty map) rather - // than trusting an assignment against a target this class cannot even identify. - val targetRelid = rangeTable[resultRelationVarno] + // 0) — never null for a real INSERT/UPDATE/DELETE/MERGE — but defensively treated as + // "substitution unsafe" rather than trusting an assignment against an unidentified target. + val targetRelid = scope.rangeTable[scope.resultRelationVarno] if (targetRelid != null && isSubstitutionSafeForRelation(targetRelid)) { nodeTreeParser.parseTargetList(nodeTree).associate { it.resultNumber to it.expression } } else { emptyMap() } } - // GROUPING SETS/CUBE/ROLLUP null-extend grouping keys AFTER the WHERE clause has already - // filtered rows, so a qual can never prove a grouped result column non-null. This matters - // even for a NOT NULL base column, because null-extension overrides the base column's own - // constraint. Suppress qual narrowing for the entire block rather than trying to map a - // (possibly expression) grouping key back to its null-extended leaf Vars — that is more - // machinery than this warrants, and a subtle mistake there would reintroduce an unsound - // narrowing. This is conservative by construction: every non-key output column of a - // grouping-sets query is an aggregate, so suppressing narrowing here costs nothing real. - // - // A non-zero resultRelationVarno (an UPDATE/DELETE/MERGE) suppresses narrowing for a similar - // reason: a qual that looks like it proves a RETURNING column non-null may in fact be testing - // the value the statement's own SET clause (or, for MERGE, an update/insert action) is about - // to overwrite. - val qualNotNullVars = if (applyQualNarrowing && !hasGroupingSets && resultRelationVarno == 0) { - NodeTreeNullabilityAnalyzer.qualProvenNonNullVars(nodeTree, isStrictFunction) - } else { - emptySet() - } val plainIsSourceColumnNotNull = { varno: Int, varattno: Int -> - if (mergeAbsentVarnos[varno] == true) { - // A MERGE relation EXPLAIN determined can be entirely absent for some result row (see - // mergeAbsentVarnos' KDoc) can never be proven non-null here, regardless of what a qual or - // this column's own catalog constraint would otherwise say — those both describe the - // relation's rows when present, which says nothing about whether this specific result row - // has one at all. - false - } else if (isProvenByQuals(qualNotNullVars, groupRteMap, varno, varattno)) { - true - } else { - val relid = rangeTable[varno] - if (relid != null) { - isColumnNotNull(relid to varattno) - } else { - val baseVar = groupRteMap[varno to varattno] - if (baseVar != null) { - val baseRelid = rangeTable[baseVar.first] - baseRelid != null && isColumnNotNull(baseRelid to baseVar.second) - } else { - subqueryColumnNotNull[varno to varattno] == true || - cteColumnNotNull[varno to varattno] == true - } - } - } + scope.isSourceColumnNotNull(varno, varattno, ::isColumnNotNull) } - val forceNewNullable = forcesNewNullable(nodeTree) - val analyzer = buildAnalyzer( - hasGroupingSets = hasGroupingSets, - forceNewNullable = forceNewNullable, - applyQualNarrowing = applyQualNarrowing, - resolvedCtes = resolvedCtes, - isSourceColumnNotNull = plainIsSourceColumnNotNull, - ) + val analyzer = buildAnalyzer(scope, depth = SUBLINK_ANALYSIS_DEPTH_BUDGET) // :returningList must be checked first, not as a fallback for an empty :targetList: an INSERT // or UPDATE's own :targetList holds the value expressions being written to each assigned - // column — a completely different, and typically shorter or differently-shaped, list than its - // RETURNING projection — so it is very often non-empty even when :returningList is what this - // call actually needs to read (`INSERT INTO t(name) VALUES ('test') RETURNING - // *` against `t(id, name)` has a one-entry :targetList for "name" alone, but a two-entry - // :returningList for "id, name"). A plain `SELECT` never populates :returningList at all, so - // this ordering only ever matters for the DML-with-RETURNING case - // [queryColumnNullabilityViaProsqlbody] reaches at the top level. + // column — a different, typically shorter list than its RETURNING projection — so it is often + // non-empty even when :returningList is what this call needs to read. `INSERT INTO t(name) + // VALUES ('test') RETURNING *` against `t(id, name)` has a one-entry :targetList for "name" + // alone, but a two-entry :returningList for "id, name". val returningEntries = nodeTreeParser.parseReturningList(nodeTree) if (returningEntries.isNotEmpty()) { // A separate analyzer whose isSourceColumnNotNull substitutes a Var referencing @@ -577,11 +504,11 @@ internal class ColumnNullabilityAnalyzer(private val loader: PgCatalogLoader) { val returningAnalyzer = buildAnalyzer( hasGroupingSets = false, - forceNewNullable = forceNewNullable, + forceNewNullable = scope.forceNewNullable, applyQualNarrowing = applyQualNarrowing, - resolvedCtes = resolvedCtes, + resolvedCtes = scope.ownCtes, ) { varno, varattno -> - val assignedExpression = if (varno == resultRelationVarno) targetListByResno[varattno] else null + val assignedExpression = if (varno == scope.resultRelationVarno) targetListByResno[varattno] else null if (assignedExpression != null) { analyzer.isNonNull(assignedExpression) } else { @@ -598,29 +525,27 @@ internal class ColumnNullabilityAnalyzer(private val loader: PgCatalogLoader) { /** * `true` when `(relid, attnum)` — [key] — is guaranteed NOT NULL, whether it identifies a base-table - * column (checked against [columnNotNullByRelidAndAttnum]) or a view column (resolved via + * column (checked against [NullabilityCatalog.columnNotNullByRelidAndAttnum]) or a view column (resolved via * [resolveViewColumnNullability], since `pg_attribute.attnotnull` is always `false` for a view * column regardless of the view's definition). * * `internal`, not `private`: [PgCatalogLoader.loadViewColumnNullability] calls this directly. */ internal fun isColumnNotNull(key: Pair): Boolean { - // A negative attribute number is a SYSTEM column (ctid, xmin, xmax, cmin, cmax, tableoid) — - // pg_attribute rows for these are typically excluded from the catalog queries that populate - // columnNotNullByRelidAndAttnum (which filters attnum > 0), so a Var referencing one would - // otherwise fall through to "not found" (nullable) even though every system column is - // unconditionally non-null for any real, returned row. + // A negative attribute number is a SYSTEM column (ctid, xmin, xmax, cmin, cmax, tableoid), + // unconditionally non-null for any real, returned row, but absent from the catalog's own + // not-null map (which only tracks attnum > 0). if (key.second < 0) return true - if (columnNotNullByRelidAndAttnum[key] == true) return true + if (catalog.columnNotNullByRelidAndAttnum[key] == true) return true val viewNullability = resolveViewColumnNullability(key.first) ?: return false return viewNullability.getOrNull(key.second - 1) == false } /** * Resolves [relid]'s per-column nullability by fully evaluating its view definition's own node - * tree (`pg_rewrite`'s `_RETURN` rule) rather than inheriting a same-named source column's - * constraint the way the `pg_depend` name-join this replaces did (`SELECT NULLIF(v, 'x') AS - * v FROM u` was reported NOT NULL whenever `u.v` was). + * tree (`pg_rewrite`'s `_RETURN` rule), rather than inheriting a same-named source column's + * constraint: `SELECT NULLIF(v, 'x') AS v FROM u` is nullable regardless of whether `u.v` is + * `NOT NULL`. * * @return one nullable flag per user-visible column (`attnum > 0 AND NOT attisdropped`), index `i` * corresponding to attnum `i + 1`, or `null` when [relid] is not a view or materialized view at all @@ -633,21 +558,20 @@ internal class ColumnNullabilityAnalyzer(private val loader: PgCatalogLoader) { internal fun resolveViewColumnNullability(relid: Int): List? { if (viewColumnNullabilityRecursionDepth >= VIEW_NULLABILITY_RECURSION_DEPTH_BUDGET) { // Depth guard — a deep but acyclic pass-through chain can exhaust the JVM stack before ever - // revisiting a relid the cycle guard below would catch. Runs before the memo lookup so a relid - // at or past the budget always truncates rather than returning a cached answer; see - // VIEW_NULLABILITY_RECURSION_DEPTH_BUDGET's KDoc. A taint event: correct only while this deep. + // revisiting a relid the cycle guard below would catch. Runs before the memo lookup so a + // relid at or past the budget always truncates rather than returning a cached answer. A + // taint event: correct only while this deep. viewColumnNullabilityTaintEventCount++ return List(columnCountFor(relid)) { true } } if (viewColumnNullabilityMemo.containsKey(relid)) { - // A memo READ of an already-tainted relid is itself a taint event for the reading frame — see - // viewColumnNullabilityTaintedRelids' KDoc. + // A memo READ of an already-tainted relid is itself a taint event for the reading frame. if (relid in viewColumnNullabilityTaintedRelids) viewColumnNullabilityTaintEventCount++ return viewColumnNullabilityMemo[relid] } if (relid in viewColumnNullabilityInProgress) { - // Cycle guard — see viewColumnNullabilityInProgress's KDoc. A taint event: this placeholder is - // correct only while the cycle is being walked, not relid's own answer. + // Cycle guard. A taint event: this placeholder is correct only while the cycle is being + // walked, not relid's own answer. viewColumnNullabilityTaintEventCount++ return List(columnCountFor(relid)) { true } } @@ -665,10 +589,9 @@ internal class ColumnNullabilityAnalyzer(private val loader: PgCatalogLoader) { } val nullability = try { if (nodeTreeParser.hasSetOperations(nodeTree)) { - // Same split as analyzeCteBodyNullability: a top-level UNION ALL/INTERSECT/EXCEPT must be - // resolved branch-by-branch and OR-combined. The synthetic name cannot collide with a CTE - // declared inside the view's own body, so previouslyResolved's self-reference entry is - // never consulted here. + // A top-level UNION ALL/INTERSECT/EXCEPT must be resolved branch-by-branch and + // OR-combined. The synthetic name cannot collide with a CTE declared inside the view's + // own body, so previouslyResolved's self-reference entry is never consulted here. analyzeSetOperationBranches(nodeTree, emptyMap(), "__norm_view_relid_$relid", applyQualNarrowing = true) } else { analyzeViewNodeTree(nodeTree) @@ -678,14 +601,14 @@ internal class ColumnNullabilityAnalyzer(private val loader: PgCatalogLoader) { } val expectedColumnCount = columnCountFor(relid) // A caught SQLException or an unanalyzable set-operation result forces the all-nullable - // fallback below; that is a taint event too, so it is not cached for this analyzer's whole + // fallback below; that is a taint event too, so it is not cached for the analyzer's whole // remaining lifetime with no eviction path. if (nullability == null || nullability.size != expectedColumnCount) { viewColumnNullabilityTaintEventCount++ } val result = alignViewColumnNullability(nullability, expectedColumnCount) - // Always memoize (see viewColumnNullabilityMemo's KDoc); whether this frame's own window saw a - // taint event decides whether relid is also marked tainted, and therefore evicted below. + // Always memoize; whether this frame's own window saw a taint event decides whether relid is + // also marked tainted, and therefore evicted below. viewColumnNullabilityMemo[relid] = result if (viewColumnNullabilityTaintEventCount != taintEventCountAtEntry) { viewColumnNullabilityTaintedRelids.add(relid) @@ -706,16 +629,14 @@ internal class ColumnNullabilityAnalyzer(private val loader: PgCatalogLoader) { } /** The number of user-visible columns (`attnum > 0 AND NOT attisdropped`) [relid] has. */ - private fun columnCountFor(relid: Int): Int = columnNotNullByRelidAndAttnum.keys.count { it.first == relid } + private fun columnCountFor(relid: Int): Int = catalog.columnNotNullByRelidAndAttnum.keys.count { it.first == relid } /** * Aligns [nullability] — one flag per non-junk target-list entry, in resno order — onto exactly * [expectedColumnCount] entries, falling back to nullable for every column when the two disagree or * when [nullability] is `null` (the analysis could not answer at all). * - * Insurance against a future PostgreSQL version breaking [resolveViewColumnNullability]'s - * resno-contiguity argument; no known real SQL reaches it. `internal`, not `private`, purely so a - * unit test can drive it with a synthetic mismatch. + * `internal`, not `private`, purely so a unit test can drive it with a synthetic mismatch. */ internal fun alignViewColumnNullability(nullability: List?, expectedColumnCount: Int): List = if (nullability != null && nullability.size == expectedColumnCount) { @@ -770,35 +691,26 @@ internal class ColumnNullabilityAnalyzer(private val loader: PgCatalogLoader) { } /** - * Creates a [NodeTreeNullabilityAnalyzer] pre-configured with this loader's catalog lookups. - * - * All constructor arguments except [isSourceColumnNotNull] are identical across every call site - * in this class. This method captures the common configuration so callers only need to supply - * the source-column resolution strategy, which varies by context (outer query, CTE body, - * subquery). + * Creates a [NodeTreeNullabilityAnalyzer] pre-configured with [catalog]'s lookups. All + * constructor arguments except [isSourceColumnNotNull] are identical across every call site in + * this class; callers only need to supply the source-column resolution strategy, which varies by + * context (outer query, CTE body, subquery). * - * @param applyQualNarrowing See [analyzeNodeTree]'s parameter of the same name — passed through - * only so [isSubLinkSubqueryColumnNotNull]'s wiring below can apply the same qual-narrowing - * policy to a `SubLink`'s subselect that the caller applies to everything else. + * @param applyQualNarrowing Passed through only so [isSubLinkSubqueryColumnNotNull]'s wiring + * below can apply the same qual-narrowing policy to a `SubLink`'s subselect that the caller + * applies to everything else. * @param depth The [subLinkSubqueryColumnNotNull] recursion budget for a `SubLink` encountered by - * the returned analyzer — see that method's KDoc for why a budget is mandatory (a subselect can - * itself contain a `SubLink`, whose own subselect can contain another). Defaults to - * [SUBLINK_ANALYSIS_DEPTH_BUDGET] for every analyzer built directly from a top-level node tree, - * CTE body, or subquery-RTE body; [subLinkSubqueryColumnNotNull] passes `depth - 1` when - * building the analyzer for a `SubLink`'s own subselect, so the budget only ever decreases - * along a chain of nested sublinks, never along the unrelated CTE/subquery-RTE recursion this - * class already performs independently of it. - * - * No `MERGE`-resolution parameter is threaded through here: a sublink's `:subselect` is, by SQL - * grammar, always a `SELECT` — a `MERGE`, `UPDATE`, or `DELETE` can never appear as a sublink's - * own subquery body — so [mergeAbsentVarnos] can never apply to anything - * [subLinkSubqueryColumnNotNull] reaches, and there is nothing for a `sql`/`EXPLAIN` parameter - * here to resolve. - * @param resolvedCtes CTE bodies declared directly in the query block this analyzer is built for — - * see [analyzeQueryBlockNullability]'s parameter of the same name for the invariant every caller - * must uphold. Threaded to [subLinkSubqueryColumnNotNull] so a `SubLink`'s subselect can resolve a - * reference to an enclosing `WITH` clause. Defaults to `emptyMap()`, the safe (nullable) - * answer for a query block with no CTEs of its own. + * the returned analyzer, since a subselect can itself contain a `SubLink`, whose own subselect + * can contain another. Defaults to [SUBLINK_ANALYSIS_DEPTH_BUDGET] for every analyzer built + * directly from a top-level node tree, CTE body, or subquery-RTE body; + * [subLinkSubqueryColumnNotNull] passes `depth - 1` when building the analyzer for a `SubLink`'s + * own subselect, so the budget only ever decreases along a chain of nested sublinks, never + * along the unrelated CTE/subquery-RTE recursion this class already performs independently. + * @param resolvedCtes CTE bodies declared directly in the query block this analyzer is built for. + * The caller must pass only CTEs declared in this exact query block, never an enclosing one's — + * threaded to [subLinkSubqueryColumnNotNull] so a `SubLink`'s subselect can resolve a reference + * to an enclosing `WITH` clause. Defaults to `emptyMap()`, the safe (nullable) answer for a + * query block with no CTEs of its own. */ private fun buildAnalyzer( hasGroupingSets: Boolean = false, @@ -808,15 +720,15 @@ internal class ColumnNullabilityAnalyzer(private val loader: PgCatalogLoader) { resolvedCtes: Map> = emptyMap(), isSourceColumnNotNull: (varno: Int, varattno: Int) -> Boolean, ): NodeTreeNullabilityAnalyzer = NodeTreeNullabilityAnalyzer( - isStrict = isStrictFunction, - hasNonNullInitialValue = { oid -> aggregateHasNonNullInitialValue[oid] == true }, + isStrict = catalog.isStrictFunction, + hasNonNullInitialValue = { oid -> catalog.aggregateHasNonNullInitialValue[oid] == true }, isSourceColumnNotNull = isSourceColumnNotNull, isOuterJoinNullable = { nullingRelations -> nullingRelations.isNotEmpty() }, - isAlwaysNonNull = { oid -> oid in alwaysNonNullFunctionOids }, - isNeverNullForNonNullInput = { oid -> oid in neverNullForNonNullInputOids }, - isLagLeadWithDefault = { oid -> oid in lagLeadWithDefaultOids }, - isFoldableToConst = { oid -> oid in immutableFunctionOids }, - isNonNullIffFirstArgumentNonNull = { oid -> oid in nonNullIffFirstArgumentNonNullFunctionOids }, + isAlwaysNonNull = { oid -> oid in catalog.alwaysNonNullFunctionOids }, + isNeverNullForNonNullInput = { oid -> oid in catalog.neverNullForNonNullInputOids }, + isLagLeadWithDefault = { oid -> oid in catalog.lagLeadWithDefaultOids }, + isFoldableToConst = { oid -> oid in catalog.immutableFunctionOids }, + isNonNullIffFirstArgumentNonNull = { oid -> oid in catalog.nonNullIffFirstArgumentNonNullFunctionOids }, isSubLinkSubqueryColumnNotNull = { subselectBlock -> subLinkSubqueryColumnNotNull(subselectBlock, applyQualNarrowing, depth, resolvedCtes) }, @@ -824,22 +736,34 @@ internal class ColumnNullabilityAnalyzer(private val loader: PgCatalogLoader) { forceNewNullable = forceNewNullable, ) + /** + * [buildAnalyzer] overload for the common case: an already-resolved [QueryBlockScope] supplies + * every argument the other overload otherwise needs spelled out at each call site — + * [QueryBlockScope.isSourceColumnNotNull], partially applied with [isColumnNotNull], for + * `isSourceColumnNotNull`; [QueryBlockScope.hasGroupingSets] and [QueryBlockScope.forceNewNullable] + * unchanged; and [QueryBlockScope.ownCtes] as `resolvedCtes`, since a `SubLink` reached from + * [scope]'s own query block can only ever resolve a CTE declared directly in it. + */ + private fun buildAnalyzer(scope: QueryBlockScope, depth: Int): NodeTreeNullabilityAnalyzer = buildAnalyzer( + hasGroupingSets = scope.hasGroupingSets, + forceNewNullable = scope.forceNewNullable, + depth = depth, + resolvedCtes = scope.ownCtes, + isSourceColumnNotNull = { varno, varattno -> scope.isSourceColumnNotNull(varno, varattno, ::isColumnNotNull) }, + ) + /** * Backs [NodeTreeNullabilityAnalyzer]'s `isSubLinkSubqueryColumnNotNull` callback: `true` when * [subselectBlock] — the raw `{QUERY ...}` text of an `ANY_SUBLINK`'s or `ALL_SUBLINK`'s * `:subselect` — produces exactly one non-junk output column and that column is provably non-null. * - * Set-operation subselects (`UNION`/`INTERSECT`/`EXCEPT`) are rejected outright, the same - * conservative default [buildSubqueryColumnNotNull] applies to a `FROM`-clause subquery RTE for - * the identical reason: tracing through would report only the first branch's nullability, not - * the union across every branch (see that method's own KDoc). + * Set-operation subselects (`UNION`/`INTERSECT`/`EXCEPT`) are rejected outright: tracing through + * would report only the first branch's nullability, not the union across every branch. * - * @param depth remaining recursion budget — see [buildAnalyzer]'s `depth` parameter KDoc. - * Returns `false` (safe: nullable) once exhausted, so a `SubLink` nested inside another - * `SubLink`'s subselect cannot recurse indefinitely. - * @param resolvedCtes See [buildAnalyzer]'s parameter of the same name — CTE bodies declared - * directly in [subselectBlock]'s own enclosing query block, so [subselectBlock] can resolve a - * reference to one of them. + * @param depth remaining recursion budget. Returns `false` (safe: nullable) once exhausted, so a + * `SubLink` nested inside another `SubLink`'s subselect cannot recurse indefinitely. + * @param resolvedCtes CTE bodies declared directly in [subselectBlock]'s own enclosing query + * block, so [subselectBlock] can resolve a reference to one of them. */ private fun subLinkSubqueryColumnNotNull( subselectBlock: String, @@ -855,69 +779,26 @@ internal class ColumnNullabilityAnalyzer(private val loader: PgCatalogLoader) { /** * `true` when a `RETURNING WITH (OLD AS o, NEW AS n)` reference to `NEW` in [nodeTree] must be - * forced nullable — see [NodeTreeNullabilityAnalyzer]'s `forceNewNullable` constructor - * parameter's KDoc for the full reasoning (a plain `DELETE` never has a `NEW` row at all; a - * `MERGE` might not, depending on which `WHEN` clause matched a given result row). + * forced nullable: a plain `DELETE` never has a `NEW` row at all; a `MERGE` might not, depending + * on which `WHEN` clause matched a given result row. */ private fun forcesNewNullable(nodeTree: String): Boolean = when (nodeTreeParser.parseCommandType(nodeTree)) { PgNodeTreeParser.COMMAND_TYPE_DELETE -> true // Only a MERGE with at least one DELETE action can leave no new row behind for some result - // row — see hasDeleteMergeAction's KDoc. A MERGE with only UPDATE/INSERT actions always - // writes or inserts a row, so NEW is exactly as trustworthy there as an ordinary column. + // row. A MERGE with only UPDATE/INSERT actions always writes or inserts a row, so NEW is + // exactly as trustworthy there as an ordinary column. PgNodeTreeParser.COMMAND_TYPE_MERGE -> nodeTreeParser.hasDeleteMergeAction(nodeTree) else -> false } - /** - * Returns `true` when `(varno, varattno)` is proven non-null by the query's `WHERE` clause, - * either directly or through the GROUP RTE remap (target-list `Var`s point at the GROUP RTE - * when `hasGroupRTE`, while `WHERE`-clause `Var`s use base relation varnos). - */ - private fun isProvenByQuals( - qualNotNullVars: Set>, - groupRteMap: Map, Pair>, - varno: Int, - varattno: Int, - ): Boolean = qualNotNullVars.contains(varno to varattno) || - groupRteMap[varno to varattno]?.let { qualNotNullVars.contains(it) } == true - - /** - * Builds a map from `(varno, varattno)` to `true` for CTE result columns that are guaranteed - * non-null. - * - * @param resolvedCtes [nodeTree]'s own directly-declared CTE bodies (from [resolveCteBodies]), - * keyed by name — passed in rather than computed here so the same resolution also feeds - * [buildAnalyzer] for a `SubLink` nested in [nodeTree]'s target list. Every CTE reference in - * [nodeTree]'s `:rtable` is `:ctelevelsup 0` by construction: [nodeTree] is always the outermost - * statement text this class analyzes, so it has no enclosing scope to point past. - */ - private fun buildCteColumnNotNull( - nodeTree: String, - resolvedCtes: Map>, - ): Map, Boolean> { - val cteRteMap = nodeTreeParser.parseCteRangeTableEntries(nodeTree) - if (cteRteMap.isEmpty()) return emptyMap() - if (resolvedCtes.isEmpty()) return emptyMap() - - return buildMap { - for ((varno, reference) in cteRteMap) { - val nullabilities = resolvedCtes[reference.name] ?: continue - nullabilities.forEachIndexed { columnIndex, nullable -> - put(varno to (columnIndex + 1), !nullable) - } - } - } - } - /** * Analyzes every CTE declared in [nodeTree]'s own `:cteList` and returns each one's per-column * nullability, keyed by CTE name. * - * Shared by [buildCteColumnNotNull] (resolving a CTE reference in [nodeTree]'s own `:rtable`) - * and [buildSubqueryColumnNotNull] (resolving a CTE reference — `:ctelevelsup 1` — one level - * down, inside a nested subquery's own `:rtable`): a CTE's declaration scope is [nodeTree]'s - * level regardless of which nesting level actually references it, so both callers resolve - * against the same set of CTE bodies. + * Shared by [buildQueryBlockScope] (resolving a CTE reference in [nodeTree]'s own `:rtable`) and + * [buildSubqueryColumnNotNull] (resolving a CTE reference — `:ctelevelsup 1` — one level down, + * inside a nested subquery's own `:rtable`): a CTE's declaration scope is [nodeTree]'s level + * regardless of which nesting level actually references it. */ private fun resolveCteBodies( nodeTree: String, @@ -935,9 +816,8 @@ internal class ColumnNullabilityAnalyzer(private val loader: PgCatalogLoader) { } /** - * @param sql See [mergeAbsentVarnos]'s parameter of the same name — passed through unchanged so - * a MERGE nested in [cte]'s own body can be resolved by the same EXPLAIN call this parameter - * documents, keyed by its own target/source relation names. + * @param sql Passed through unchanged so a `MERGE` nested in [cte]'s own body can be resolved + * via the same `EXPLAIN` call, keyed by its own target/source relation names. */ private fun analyzeCteBodyNullability( cte: NodeTreeCteDefinition, @@ -948,17 +828,13 @@ internal class ColumnNullabilityAnalyzer(private val loader: PgCatalogLoader) { if (nodeTreeParser.hasSetOperations(cte.queryBlock)) { return analyzeSetOperationBranches(cte.queryBlock, previouslyResolved, cte.name, applyQualNarrowing) } - val cteRangeTable = nodeTreeParser.parseRangeTable(cte.queryBlock) + val cteRangeTable = nodeTreeParser.parseRangeTableEntries(cte.queryBlock).baseRelations() val mergeAbsent = mergeAbsentVarnos(cte.queryBlock, cteRangeTable, sql) ?: return null val analyzer = buildCteBodyAnalyzer(cte.queryBlock, previouslyResolved, applyQualNarrowing, mergeAbsent, sql) - // :returningList must be checked first, not as a fallback for an empty :targetList — see - // analyzeNodeTree's identical guard for the full reasoning (an INSERT/UPDATE's own :targetList - // holds the value expressions being written, a completely different list from its RETURNING - // projection, and is very often non-empty even when :returningList is what must be read). A - // data-modifying CTE body reaches this method with its raw, un-rewritten :returningList only - // via [queryColumnNullabilityViaProsqlbody] — `prosqlbody` is the only mechanism that ever - // populates a node tree for a data-modifying CTE in the first place (see that function's - // KDoc), so there is no other caller shape this ordering needs to account for. + // :returningList must be checked first, not as a fallback for an empty :targetList: an + // INSERT/UPDATE's own :targetList holds the value expressions being written, a different list + // from its RETURNING projection, and is often non-empty even when :returningList is what must + // be read. val returningEntries = nodeTreeParser.parseReturningList(cte.queryBlock) if (returningEntries.isNotEmpty()) { return returningEntries @@ -975,29 +851,21 @@ internal class ColumnNullabilityAnalyzer(private val loader: PgCatalogLoader) { * each branch's per-column nullability with OR: a column is nullable in the combined result if * any branch can produce `null` for it. * - * A `WITH RECURSIVE` CTE's recursive term(s) reference the CTE by name ([cteName]) — resolved by - * [buildCteBodyAnalyzer] via `previouslyResolved` — creating a genuine fixpoint problem: the - * recursive term's own nullability depends on the CTE's own combined nullability, which this - * function is what computes. PostgreSQL requires the first branch (the seed/non-recursive term) - * to never reference the CTE itself, so it alone is computed once, outside the loop, as a known - * starting point. Every subsequent branch (there is always exactly one recursive term for a - * `WITH RECURSIVE` CTE, but this handles a plain multi-branch `UNION` identically) is then - * re-analyzed, feeding back the current combined result as [cteName]'s own nullability, and the - * combined result is recomputed — repeated until a pass changes nothing. + * A `WITH RECURSIVE` CTE's recursive term(s) reference the CTE by name ([cteName]), creating a + * genuine fixpoint problem: the recursive term's own nullability depends on the CTE's own + * combined nullability, which this function computes. The seed (first) branch never references + * the CTE itself — PostgreSQL requires this — so it is computed once, outside the loop, as a + * known starting point. Every subsequent branch is then re-analyzed, feeding back the current + * combined result as [cteName]'s own nullability, and the combined result is recomputed — + * repeated until a pass changes nothing. * - * This converges because the per-column nullability lattice (`false` = NOT NULL, `true` = - * nullable, ordered `false < true`) is monotone under this loop's own update rule: OR-combining - * more branch results (now including a possibly-wider self-reference) can only ever add `true` - * bits, never remove one. Starting from the seed's own (fixed, correct) nullability — the - * narrowest value the CTE's self-reference could possibly have — and iterating a - * monotone-widening step over a `columnCount`-bit lattice reaches its fixpoint in at most - * `columnCount + 1` passes (each pass either flips at least one more bit `false → true`, or - * changes nothing and the loop stops), which the `while` condition below checks directly rather - * than trusting an iteration-count bound alone. + * This converges because OR-combining more branch results can only ever add `true` (nullable) + * bits, never remove one: a monotone-widening step over a `columnCount`-bit lattice reaches its + * fixpoint in at most `columnCount + 1` passes, which the `while` condition below checks directly. * * @return `null` if [queryBlock] has no analyzable subquery branches at all, or if the seed - * branch itself could not be analyzed (an empty result — see [extractColumnNullability]'s - * contract). Otherwise, one nullability value per output column, in `SELECT` order. + * branch itself could not be analyzed (an empty result). Otherwise, one nullability value per + * output column, in `SELECT` order. */ private fun analyzeSetOperationBranches( queryBlock: String, @@ -1005,12 +873,11 @@ internal class ColumnNullabilityAnalyzer(private val loader: PgCatalogLoader) { cteName: String, applyQualNarrowing: Boolean = true, ): List? { - val subqueryBranches = nodeTreeParser.parseSubqueryRangeTable(queryBlock).values.toList() + val subqueryBranches = nodeTreeParser.parseRangeTableEntries(queryBlock).subqueryBlocks().values.toList() if (subqueryBranches.isEmpty()) return null - // The seed (first) branch of a recursive CTE structurally cannot reference the CTE itself — - // PostgreSQL rejects a self-reference in the non-recursive term — so its nullability never - // depends on the fixpoint loop below and is computed exactly once. + // The seed (first) branch of a recursive CTE structurally cannot reference the CTE itself, so + // its nullability never depends on the fixpoint loop below and is computed exactly once. val seedBlock = subqueryBranches.first() val seedAnalyzer = buildCteBodyAnalyzer(seedBlock, previouslyResolved, applyQualNarrowing) val seedResult = seedAnalyzer.extractColumnNullability(seedBlock) @@ -1020,11 +887,8 @@ internal class ColumnNullabilityAnalyzer(private val loader: PgCatalogLoader) { if (otherBranches.isEmpty()) return seedResult // Bounded at columnCount + 1 passes — the maximum this monotone-widening loop can take to - // reach its fixpoint (see this function's own KDoc for the termination argument) — rather - // than trusting that argument alone to keep this loop from ever running forever. columnCount - // itself is seedResult.size, since every branch of a UNION/set operation is required (by - // PostgreSQL) to have the same column count as every other branch, and the seed's is already - // known at this point. + // reach its fixpoint. columnCount itself is seedResult.size, since every branch of a UNION/set + // operation is required (by PostgreSQL) to have the same column count as every other branch. var combined = seedResult var previous: List? var remainingPasses = seedResult.size + 1 @@ -1038,166 +902,143 @@ internal class ColumnNullabilityAnalyzer(private val loader: PgCatalogLoader) { val result = branchAnalyzer.extractColumnNullability(branchBlock) // An empty result means this branch's own nullability could not be determined at all — not // "this branch has zero columns" (impossible; every branch of a set operation has the same - // column count). Silently dropping it from the OR-combination would let the other - // branches' (possibly narrower, even all-NOT-NULL) answer stand as if this branch - // contributed nothing, when in truth its contribution is simply unknown. Flagging it here - // forces every column nullable below instead. + // column count). Flagging it forces every column nullable below instead of silently + // dropping it and letting the other branches' answer stand as if it contributed nothing. if (result.isNotEmpty()) branchResults.add(result) else anyBranchUnanalyzable = true } val columnCount = branchResults.maxOf { it.size } combined = (0 until columnCount).map { col -> branchResults.any { it.getOrElse(col) { true } } } remainingPasses-- } while (combined != previous && remainingPasses > 0) - // combined != previous here means the loop was cut off by remainingPasses running out while - // the result was still changing — i.e. it did not reach its fixpoint. Returning that - // still-moving intermediate value could under-report nullability (a later pass might still - // flip more columns to nullable), so every column is forced nullable instead, the same - // fallback taken for an unanalyzable branch above. + // combined != previous here means the loop was cut off before reaching its fixpoint. Returning + // that still-moving intermediate value could under-report nullability, so every column is + // forced nullable instead, the same fallback taken for an unanalyzable branch above. val didNotConverge = combined != previous return if (anyBranchUnanalyzable || didNotConverge) List(combined.size) { true } else combined } /** - * Builds a map from `(varno, varattno)` to `true` for CTE RTE columns that are non-null. + * Resolves everything [QueryBlockScope.isSourceColumnNotNull] needs to answer a `Var` reference + * inside [queryBlock] — the source-column-resolution chain [analyzeNodeTree], + * [buildCteBodyAnalyzer], and [analyzeQueryBlockNullability] all build against. * - * `:ctelevelsup 0` means [queryBlock] declares that CTE itself, possibly shadowing a sibling of the - * same name one level up, so it resolves from [ownResolvedCtes]; anything greater resolves from - * [previouslyResolved]. Without this split a local shadowing `WITH` resolved against the wrong - * sibling body — an unsound answer, not merely a widened one. + * `:ctelevelsup 0` means [queryBlock] declares that CTE reference's own CTE, possibly shadowing a + * sibling of the same name one level up, so it resolves from [enclosingCtes]'s own-scope + * counterpart — the freshly-resolved [QueryBlockScope.ownCtes] — rather than [enclosingCtes] + * itself; anything greater resolves from [enclosingCtes]. Without this split a local shadowing + * `WITH` would resolve against the wrong sibling body — an unsound answer, not merely a widened + * one. * - * @param ownResolvedCtes CTE bodies declared directly in [queryBlock]'s own `:cteList`. - * @param previouslyResolved CTE bodies declared in the same outer `:cteList` [queryBlock]'s own CTE - * is declared in (siblings declared earlier in that `WITH` clause). - */ - private fun buildInnerCteNotNull( - queryBlock: String, - ownResolvedCtes: Map>, - previouslyResolved: Map>, - ): Map, Boolean> { - val innerCteRtes = nodeTreeParser.parseCteRangeTableEntries(queryBlock) - return buildMap { - for ((varno, reference) in innerCteRtes) { - val ctesInScope = if (reference.ctelevelsup == 0) ownResolvedCtes else previouslyResolved - val nullabilities = ctesInScope[reference.name] ?: continue - nullabilities.forEachIndexed { columnIndex, nullable -> - put(varno to (columnIndex + 1), !nullable) - } - } - } - } - - /** - * @param applyQualNarrowing See [analyzeNodeTree]'s parameter of the same name. - * @param mergeAbsentVarnos See [analyzeNodeTree]'s parameter of the same name — [queryBlock]'s - * own varno-to-canBeAbsent map when [queryBlock] itself is a `MERGE` (resolved by - * [analyzeCteBodyNullability] before ever calling this method), empty otherwise. - * @param sql See [mergeAbsentVarnos]'s (the method, not this parameter) `sql` parameter — passed - * through only so a subquery within [queryBlock] that references a CTE declared in - * [queryBlock]'s own nested `WITH` clause can resolve that (deeper) CTE's `MERGE`, if it has - * one, through [buildSubqueryColumnNotNull]. Defaults to an empty string for the (`SELECT`-only, - * never `MERGE`-shaped) set-operation branch callers in [analyzeSetOperationBranches], where an - * empty `EXPLAIN` target simply fails harmlessly (caught, treated as "cannot resolve") for the - * narrow, deeper case of a subquery nested that deep referencing its own local `MERGE` CTE. + * @param enclosingCtes CTE bodies visible via `:ctelevelsup` greater than `0` relative to + * [queryBlock] — declared in whichever scope encloses it, never [queryBlock]'s own nested `WITH` + * clause. Empty for [queryBlock]'s outermost statement, which has no enclosing scope to point + * past. + * @param applyQualNarrowing Gates [QueryBlockScope.qualProvenVars]: suppressed whenever + * [queryBlock] has GROUPING SETS/CUBE/ROLLUP (which null-extend a grouping key AFTER `WHERE` + * has already filtered rows) or is itself a data-modifying statement (a non-zero + * `:resultRelation`) — `WITH c AS (UPDATE t SET a = NULL FROM u WHERE u.id = t.id AND t.a IS + * NOT NULL RETURNING t.a) SELECT a FROM c` returns `a = NULL`, not the value the `WHERE` + * clause proved before the `SET` ran. + * @param sql Passed through only so a data-modifying CTE nested inside [queryBlock]'s own `WITH` + * clause can resolve its own `MERGE` via the same `EXPLAIN` call. Defaults to an empty string + * for the set-operation branch callers, where an empty `EXPLAIN` target simply fails + * harmlessly (caught, treated as "cannot resolve"). + * @param depth The [subLinkSubqueryColumnNotNull] recursion budget threaded, not refilled, + * through a recursive hop into a nested query block. + * @param mergeAbsentVarnos [queryBlock]'s own varno-to-canBeAbsent map when [queryBlock] itself is + * a `MERGE` — empty for every query block that cannot itself be one (a `SELECT`'s `FROM` + * subquery, a `SubLink`'s subselect, or a set-operation branch). */ - private fun buildCteBodyAnalyzer( + private fun buildQueryBlockScope( queryBlock: String, - previouslyResolved: Map>, - applyQualNarrowing: Boolean = true, + enclosingCtes: Map>, + applyQualNarrowing: Boolean, + @Language("PostgreSQL") sql: String, + depth: Int = SUBLINK_ANALYSIS_DEPTH_BUDGET, mergeAbsentVarnos: Map = emptyMap(), - @Language("PostgreSQL") sql: String = "", - ): NodeTreeNullabilityAnalyzer { - val cteRangeTable = nodeTreeParser.parseRangeTable(queryBlock) - // See analyzeNodeTree's identical guard: GROUPING SETS/CUBE/ROLLUP can null-extend a - // grouping key even when the underlying base table column is NOT NULL, and even when the - // WHERE clause proved it non-null before aggregation. The actual override (including - // EXPRESSION grouping keys) is applied by extractColumnNullability via hasGroupingSets below. + ): QueryBlockScope { + val rangeTableEntries = nodeTreeParser.parseRangeTableEntries(queryBlock) + val rangeTable = rangeTableEntries.baseRelations() val hasGroupingSets = nodeTreeParser.hasGroupingSets(queryBlock) val groupRteMap = if (hasGroupingSets) { emptyMap() } else { - nodeTreeParser.parseGroupRteMap(queryBlock) + rangeTableEntries.groupRteMap(nodeTreeParser) } - // queryBlock's own nested WITH clause, distinct from previouslyResolved (sibling CTEs one level - // further up). Resolving any of the three uses below against previouslyResolved instead would - // resolve a shadowing local WITH against the wrong body. - val ownResolvedCtes = resolveCteBodies(queryBlock, applyQualNarrowing, sql) - val innerCteNotNull = buildInnerCteNotNull(queryBlock, ownResolvedCtes, previouslyResolved) - val subqueryColumnNotNull = buildSubqueryColumnNotNull(queryBlock, ownResolvedCtes, applyQualNarrowing, sql) - // See analyzeNodeTree's identical guard for why qual narrowing is suppressed whenever - // hasGroupingSets: a grouping key is exactly the thing a GROUPING SETS/CUBE/ROLLUP query - // null-extends after WHERE has already run. A non-zero :resultRelation suppresses narrowing - // for the identical reason analyzeNodeTree's own guard does: a data-modifying CTE body's WHERE - // clause can prove something about a column its own SET clause is about to overwrite — see - // e.g. `WITH c AS (UPDATE t SET a = NULL FROM u WHERE u.id = t.id AND t.a IS NOT NULL - // RETURNING t.a) SELECT a FROM c` returns `a = NULL`, not the - // value the WHERE clause proved before the SET ran. - val isDml = nodeTreeParser.parseResultRelation(queryBlock) != 0 - val qualNotNullVars = if (applyQualNarrowing && !hasGroupingSets && !isDml) { - NodeTreeNullabilityAnalyzer.qualProvenNonNullVars(queryBlock, isStrictFunction) + val ownCtes = resolveCteBodies(queryBlock, applyQualNarrowing, sql) + val subqueryColumnNotNull = buildSubqueryColumnNotNull(queryBlock, ownCtes, applyQualNarrowing, sql, depth) + val cteReferences = rangeTableEntries.cteReferences() + val resultRelationVarno = nodeTreeParser.parseResultRelation(queryBlock) + val qualProvenVars = if (applyQualNarrowing && !hasGroupingSets && resultRelationVarno == 0) { + NodeTreeNullabilityAnalyzer.qualProvenNonNullVars(queryBlock, catalog.isStrictFunction) } else { emptySet() } - return buildAnalyzer( + return QueryBlockScope( + rangeTable = rangeTable, hasGroupingSets = hasGroupingSets, + groupRteMap = groupRteMap, + qualProvenVars = qualProvenVars, + ownCtes = ownCtes, + enclosingCtes = enclosingCtes, + cteReferences = cteReferences, + subqueryColumnNotNull = subqueryColumnNotNull, + mergeAbsentVarnos = mergeAbsentVarnos, forceNewNullable = forcesNewNullable(queryBlock), - applyQualNarrowing = applyQualNarrowing, - resolvedCtes = ownResolvedCtes, - ) { - varno, - varattno, - -> - if (mergeAbsentVarnos[varno] == true) { - // See analyzeNodeTree's identical guard: a MERGE relation EXPLAIN determined can be - // entirely absent for some result row can never be proven non-null here. - false - } else if (isProvenByQuals(qualNotNullVars, groupRteMap, varno, varattno)) { - true - } else { - val relid = cteRangeTable[varno] - if (relid != null) { - isColumnNotNull(relid to varattno) - } else { - val baseVar = groupRteMap[varno to varattno] - if (baseVar != null) { - val baseRelid = cteRangeTable[baseVar.first] - baseRelid != null && isColumnNotNull(baseRelid to baseVar.second) - } else { - subqueryColumnNotNull[varno to varattno] == true || - innerCteNotNull[varno to varattno] == true - } - } - } - } + resultRelationVarno = resultRelationVarno, + ) } + /** + * @param mergeAbsentVarnos [queryBlock]'s own varno-to-canBeAbsent map when [queryBlock] itself + * is a `MERGE` (resolved by [analyzeCteBodyNullability] before ever calling this method), + * empty otherwise. + * @param sql Passed through only so a subquery within [queryBlock] that references a CTE + * declared in [queryBlock]'s own nested `WITH` clause can resolve that (deeper) CTE's + * `MERGE`, if it has one, through [buildSubqueryColumnNotNull]. Defaults to an empty string + * for the set-operation branch callers, where an empty `EXPLAIN` target simply fails + * harmlessly. + */ + private fun buildCteBodyAnalyzer( + queryBlock: String, + previouslyResolved: Map>, + applyQualNarrowing: Boolean = true, + mergeAbsentVarnos: Map = emptyMap(), + @Language("PostgreSQL") sql: String = "", + ): NodeTreeNullabilityAnalyzer = buildAnalyzer( + buildQueryBlockScope( + queryBlock, + previouslyResolved, + applyQualNarrowing, + sql, + mergeAbsentVarnos = mergeAbsentVarnos, + ), + depth = SUBLINK_ANALYSIS_DEPTH_BUDGET, + ) + /** * Builds a map from `(varno, varattno)` to `true` for columns of subquery RTEs that are * guaranteed non-null. * * For each `rtekind 1` (subquery) entry in the outer query's `:rtable`, extracts the embedded - * `:subquery {QUERY ...}` block, recursively analyzes it with a fresh [NodeTreeNullabilityAnalyzer] - * using the **subquery's own range table**, and maps each output column position to its nullability. - * - * This allows the outer analyzer's [NodeTreeNullabilityAnalyzer] to correctly evaluate VARs that - * reference a subquery derived table (`SELECT s.col FROM (...) s`) rather than a base table. - * Without this, `isSourceColumnNotNull` for a subquery VAR would always return `false` (nullable) - * because the subquery RTE has no `relid` in the outer range table. + * `:subquery {QUERY ...}` block, recursively analyzes it with the subquery's own range table, + * and maps each output column position to its nullability. This lets the outer analyzer + * correctly evaluate a VAR referencing a subquery derived table (`SELECT s.col FROM (...) s`), + * which otherwise has no `relid` in the outer range table and would always resolve nullable. * * @param nodeTree the `pg_rewrite.ev_action` text of the outer query's temporary view, or a * nested `{QUERY ...}` block reached via [analyzeQueryBlockNullability] (a `SubLink`'s own * subselect, or a derived table nested inside one) * @param resolvedCtes CTE bodies declared directly in [nodeTree]'s own `:cteList`. A subquery * nested inside [nodeTree] can reference one of these via `:ctelevelsup 1` inside its own - * `:rtable`, not [nodeTree]'s. Computed by every caller so the same resolution also feeds - * [buildAnalyzer] for a `SubLink` nested in [nodeTree]. - * @param applyQualNarrowing See [analyzeNodeTree]'s parameter of the same name. + * `:rtable`, not [nodeTree]'s. * @param depth The [subLinkSubqueryColumnNotNull] recursion budget, threaded to * [analyzeQueryBlockNullability] for a `SubLink` reached via one of [nodeTree]'s subquery RTEs. * Callers resolving [nodeTree]'s own top-level subquery RTEs use the default, full budget — a * `FROM`-clause hop is not a nested-sublink hop. [analyzeQueryBlockNullability]'s own recursive - * call passes its current, possibly already-decremented `depth` through unchanged: it is the only - * path reaching a derived table nested inside a `SubLink`'s subselect, and refilling the budget - * there would let a chain of `= ANY` sublinks separated by derived tables bypass it. + * call passes its current, possibly already-decremented `depth` through unchanged, since + * refilling it there would let a chain of `= ANY` sublinks separated by derived tables bypass + * the budget. * @return A map from `(varno, varattno)` pairs to `true` when the subquery column is non-null */ private fun buildSubqueryColumnNotNull( @@ -1209,12 +1050,10 @@ internal class ColumnNullabilityAnalyzer(private val loader: PgCatalogLoader) { ): Map, Boolean> { // Set-operation queries (UNION ALL, INTERSECT, EXCEPT) store their branches as rtekind=1 // subquery RTEs. Tracing through them would incorrectly report the first branch's nullability - // as the result's nullability — the true result is the union across all branches, some of which - // may introduce nulls (e.g., a branch with LEFT JOIN). Return empty so the analyzer conservatively - // treats set-operation output columns as nullable (the correct safe default). This also covers - // analyzeQueryBlockNullability's recursive call into this method. + // as the result's nullability, so this returns empty, conservatively treating set-operation + // output columns as nullable. if (nodeTreeParser.hasSetOperations(nodeTree)) return emptyMap() - val subqueryRangeTable = nodeTreeParser.parseSubqueryRangeTable(nodeTree) + val subqueryRangeTable = nodeTreeParser.parseRangeTableEntries(nodeTree).subqueryBlocks() if (subqueryRangeTable.isEmpty()) return emptyMap() return buildMap { for ((outerVarno, subqueryBlock) in subqueryRangeTable) { @@ -1229,30 +1068,20 @@ internal class ColumnNullabilityAnalyzer(private val loader: PgCatalogLoader) { } /** - * Computes per-column nullability for a single query block ([queryBlock]) — the shared core of - * [buildSubqueryColumnNotNull] (a `FROM`-clause subquery RTE) and [subLinkSubqueryColumnNotNull] - * (an `ANY_SUBLINK`'s or `ALL_SUBLINK`'s `:subselect`): given a raw `{QUERY ...}` block, build a resolver over the - * block's own `parseRangeTable`/`parseCteRangeTableEntries`/`parseGroupRteMap`/subquery-RTE/qual - * narrowing, then run [NodeTreeNullabilityAnalyzer.extractColumnNullability] against it. - * - * [queryBlock]'s own `:rtable` can hold three things resolved differently: a base table (via - * [isColumnNotNull]), a nested subquery RTE (a derived table, resolved by recursing into - * [buildSubqueryColumnNotNull] on [queryBlock] itself), and a CTE RTE. Before this fix only the - * base-table case was handled, so a `SubLink`'s subselect reading either of the others degraded to - * nullable. + * Computes per-column nullability for a single query block ([queryBlock]): builds a + * [QueryBlockScope] for it and runs [NodeTreeNullabilityAnalyzer.extractColumnNullability] + * against that scope. * * @param resolvedCtes CTE bodies visible via `:ctelevelsup` greater than `0` relative to * [queryBlock] — declared in whichever scope encloses it, never [queryBlock]'s own nested `WITH` - * clause. Every caller must uphold this; resolving a `Var` against the wrong CTE body is silently - * worse than widening — see [NodeTreeCteReference.ctelevelsup]. A CTE at `:ctelevelsup 2` (a - * sibling of [queryBlock]'s own enclosing CTE) is out of this flat map's reach and stays - * conservatively nullable; reaching it would mean threading a stack of scopes through every caller. - * @param depth See [buildAnalyzer]'s parameter of the same name — passed through unchanged so a - * `SubLink` inside [queryBlock], and this method's own [buildSubqueryColumnNotNull] call for a - * derived table nested inside it, both get the already-decremented budget. - * @param sql See [mergeAbsentVarnos]'s `sql` parameter — passed through only so a data-modifying CTE - * in [queryBlock]'s own nested `WITH` clause can resolve its `MERGE`. The empty-string default - * makes that `EXPLAIN` fail harmlessly, the same limitation [buildCteBodyAnalyzer]'s default has. + * clause. The caller must uphold this; resolving a `Var` against the wrong CTE body is unsound, + * not merely widened. + * @param depth Passed through unchanged so a `SubLink` inside [queryBlock], and this method's own + * [buildSubqueryColumnNotNull] call for a derived table nested inside it, both get the + * already-decremented budget. + * @param sql Passed through only so a data-modifying CTE in [queryBlock]'s own nested `WITH` + * clause can resolve its `MERGE`. The empty-string default makes that `EXPLAIN` fail + * harmlessly. */ private fun analyzeQueryBlockNullability( queryBlock: String, @@ -1260,60 +1089,9 @@ internal class ColumnNullabilityAnalyzer(private val loader: PgCatalogLoader) { resolvedCtes: Map>, depth: Int = SUBLINK_ANALYSIS_DEPTH_BUDGET, @Language("PostgreSQL") sql: String = "", - ): List { - // Parse the block's own base-table range table for isSourceColumnNotNull. - val subRangeTable = nodeTreeParser.parseRangeTable(queryBlock) - val subCteRteMap = nodeTreeParser.parseCteRangeTableEntries(queryBlock) - // See analyzeNodeTree's identical guard: GROUPING SETS/CUBE/ROLLUP can - // null-extend a grouping key even when the base table column is NOT NULL. The actual - // override (including EXPRESSION grouping keys) is applied by extractColumnNullability - // via hasGroupingSets below. - val hasGroupingSets = nodeTreeParser.hasGroupingSets(queryBlock) - val groupRteMap = if (hasGroupingSets) { - emptyMap() - } else { - nodeTreeParser.parseGroupRteMap(queryBlock) - } - // See analyzeNodeTree's identical guard: suppress qual narrowing whenever - // hasGroupingSets, because those are exactly what GROUPING SETS/CUBE/ROLLUP null-extends - // after WHERE has already filtered rows. - val subQualNotNullVars = if (applyQualNarrowing && !hasGroupingSets) { - NodeTreeNullabilityAnalyzer.qualProvenNonNullVars(queryBlock, isStrictFunction) - } else { - emptySet() - } - // A ctelevelsup-0 reference must resolve against queryBlock's own CTEs, never the resolvedCtes - // parameter, which belongs to an enclosing scope. - val ownResolvedCtes = resolveCteBodies(queryBlock, applyQualNarrowing, sql) - // depth is threaded, not defaulted: this is the recursive hop that must not refill the budget. - val subqueryColumnNotNull = buildSubqueryColumnNotNull(queryBlock, ownResolvedCtes, applyQualNarrowing, sql, depth) - val subAnalyzer = buildAnalyzer( - hasGroupingSets = hasGroupingSets, - applyQualNarrowing = applyQualNarrowing, - depth = depth, - resolvedCtes = ownResolvedCtes, - ) { subVarno, subVarattno -> - if (isProvenByQuals(subQualNotNullVars, groupRteMap, subVarno, subVarattno)) { - true - } else { - val relid = subRangeTable[subVarno] - if (relid != null) { - isColumnNotNull(relid to subVarattno) - } else if (subqueryColumnNotNull[subVarno to subVarattno] == true) { - true - } else { - val cteReference = subCteRteMap[subVarno] - if (cteReference == null) { - false - } else { - val ctesInScope = if (cteReference.ctelevelsup == 0) ownResolvedCtes else resolvedCtes - ctesInScope[cteReference.name]?.getOrNull(subVarattno - 1) == false - } - } - } - } - return subAnalyzer.extractColumnNullability(queryBlock) - } + ): List = + buildAnalyzer(buildQueryBlockScope(queryBlock, resolvedCtes, applyQualNarrowing, sql, depth), depth) + .extractColumnNullability(queryBlock) /** * Answers, for [analyzeNodeTree]'s own `:targetList`-to-`:returningList` substitution, whether a @@ -1374,14 +1152,12 @@ internal class ColumnNullabilityAnalyzer(private val loader: PgCatalogLoader) { } /** - * The bare (unqualified) table name for [relid], via `pg_class.relname` — used to attribute - * an `EXPLAIN` plan's `"Relation Name"` fields (which are always the real table name, never an - * alias) back to a specific `:rtable` entry this class already resolved structurally, without - * ever re-parsing the SQL text for a table name or alias. See [mergeAbsentVarnos]'s only caller. + * The bare (unqualified) table name for [relid], via `pg_class.relname` — used to attribute an + * `EXPLAIN` plan's `"Relation Name"` fields (always the real table name, never an alias) back to + * a specific `:rtable` entry. * - * @return `null` if [relid] cannot be resolved (should not happen for a real, structural - * `:rtable` entry, but treated the same as any other "cannot confirm" case: the caller must - * fall back to its own safe default rather than guess) + * @return `null` if [relid] cannot be resolved; the caller must fall back to its own safe + * default rather than guess. */ private fun resolveTableName(relid: Int): String? = try { connection.prepareStatement("SELECT relname FROM pg_catalog.pg_class WHERE oid = ?").use { preparedStatement -> diff --git a/generator/src/main/kotlin/norm/generator/DomainBuilder.kt b/generator/src/main/kotlin/norm/generator/DomainBuilder.kt index 60b6776b..ab9e7ab3 100644 --- a/generator/src/main/kotlin/norm/generator/DomainBuilder.kt +++ b/generator/src/main/kotlin/norm/generator/DomainBuilder.kt @@ -122,20 +122,20 @@ internal fun domainAdapterPropertyName(domain: Domain): String = "${domain.name. /** * Maps a Postgres base type name to the corresponding Kotlin [TypeName]. * - * Delegates to [resolveJdbcTypeInfo] as the single source of truth for type mappings. Every type + * Delegates to [resolveWireCodec] as the single source of truth for type mappings. Every type * with an entry there is usable as a domain base, including `json` and `jsonb`: the value class * wraps the same Kotlin type the plain column would produce, and the binding difference - * (`setObject(..., Types.OTHER)` rather than `setString`) is carried by the [JdbcTypeInfo] that + * (`setObject(..., Types.OTHER)` rather than `setString`) is carried by the [WireCodec] that * [TypeRepository] hands to [AdaptedTypeSqlMappable], not by the wrapped Kotlin type. * - * [resolveJdbcTypeInfo] has an entry for every type [TypeRepository]'s `BASE_TYPE_RESOLVERS` - * accepts as a plain column type (enforced by [ColumnTypeMappingTest]'s domain-base-type-parity - * sweep), so [error] here is unreachable for a domain built on any of those — `CREATE DOMAIN d AS - * timestamptz`/`uuid`/`date`/etc. all resolve. It stays reachable for a Postgres type Norm has + * [resolveWireCodec] reads [POSTGRES_BASE_TYPES], the same map [TypeRepository.resolveBaseType] + * reads for a plain column's type, so [error] here is unreachable for a domain built on any type + * that map supports — `CREATE DOMAIN d AS timestamptz`/`uuid`/`date`/etc. all resolve, by + * construction. It stays reachable for a Postgres type Norm has * never mapped to Kotlin at all, as a plain column or otherwise (`xml`, `interval`, `money`, ...). * Postgres allows a domain over any of these, so hitting this is expected, not a bug — failing * fast with the unsupported type's name beats silently guessing a mapping Norm has no tested * behavior for. */ internal fun domainKotlinBaseType(baseTypeName: String): TypeName = - resolveJdbcTypeInfo(baseTypeName)?.kotlinType ?: error("Unsupported domain base type: $baseTypeName") + resolveWireCodec(baseTypeName)?.kotlinType ?: error("Unsupported domain base type: $baseTypeName") diff --git a/generator/src/main/kotlin/norm/generator/GroupRteSubstitution.kt b/generator/src/main/kotlin/norm/generator/GroupRteSubstitution.kt index a2b44de5..02430881 100644 --- a/generator/src/main/kotlin/norm/generator/GroupRteSubstitution.kt +++ b/generator/src/main/kotlin/norm/generator/GroupRteSubstitution.kt @@ -4,7 +4,7 @@ import norm.generator.NodeTreeNullabilityAnalyzer.Companion.MAX_EXPRESSION_DEPTH /** * Substitutes every [PgNodeExpression.Var] in [expression] that references a PostgreSQL 18+ GROUP - * RTE (see [PgNodeTreeParser.parseGroupRteExpressions]) with the resolved `:groupexprs` expression + * RTE (see [groupExpressions]) with the resolved `:groupexprs` expression * it stands in for, restoring the same tree shape PostgreSQL 16 and 17 produce directly (where the * original grouping-key expression is left in the target list, never masked behind a `Var`). This * lets [NodeTreeNullabilityAnalyzer] apply one set of nullability rules to every supported @@ -20,7 +20,7 @@ import norm.generator.NodeTreeNullabilityAnalyzer.Companion.MAX_EXPRESSION_DEPTH * shape this parser does not (yet) model correctly, and substituting against a nonexistent entry * would silently invent an expression PostgreSQL never produced. * - The resolved expression is not [PgNodeExpression.Unknown] — an unmodelled node type or a parse - * failure must not replace a `Var` that [PgNodeTreeParser.parseGroupRteMap]'s coarser, `Var`-only + * failure must not replace a `Var` that [groupRteMap]'s coarser, `Var`-only * resolution could still succeed at (see that method's continued use as a fallback wherever this * substitution declines to apply). * @@ -40,28 +40,10 @@ import norm.generator.NodeTreeNullabilityAnalyzer.Companion.MAX_EXPRESSION_DEPTH * GROUP-RTE-shaped) structurally unable to loop, without this function ever having to prove * PostgreSQL cannot emit such a cycle — a proof this function does not attempt. * - * Child coverage mirrors [NodeTreeNullabilityAnalyzer.containsVarOutsideRelation], not - * [NodeTreeNullabilityAnalyzer.safetyWalkChildren]: the latter drops several children (e.g. - * [PgNodeExpression.Aggref]'s own arguments, [PgNodeExpression.JsonExpr]'s `PASSING`-adjacent - * fields, everything past [PgNodeExpression.SubLink.outerOperand]) for reasons specific to the - * grouping-set safety analysis it backs, which do not apply here. A GROUP RTE `Var` can appear - * buried inside a non-`Aggref` node this substitution must walk into — e.g. `count(*) + - * 0::bigint` when `0::bigint` is also the grouping key rewrites that operand of the `+` to a GROUP - * RTE `Var`, sitting alongside the `Aggref` as an [PgNodeExpression.OpExpr] argument. - * [PgNodeExpression.Aggref]'s own arguments are a different case: PostgreSQL never rewrites them at - * all, on PG18 or otherwise — `count(lower(a))`/`string_agg(lower(a), ',')` under - * `GROUP BY ROLLUP(lower(a))` keep the real base-relation `Var` inside the `AGGREF`'s `:args`, - * because aggregate arguments are evaluated pre-grouping, before any GROUP RTE substitution could - * apply. This substitution still walks into `Aggref`'s arguments anyway — not because any known PG18 - * shape needs it, but defensively: the exhaustive `when` below requires some branch for `Aggref` - * regardless, and doing the real rewrite there (rather than treating it as a childless leaf) means a - * future PostgreSQL shape, or an unrelated caller that hands this function an `Aggref` subtree - * directly, cannot silently escape substitution. Every child this substitution's `when` retains must - * be rewritten, or a buried GROUP RTE `Var` would silently survive un-substituted. - * - * The `when` below is exhaustive over the sealed [PgNodeExpression] hierarchy with no `else ->` - * branch: adding a new [PgNodeExpression] subtype without updating this function fails the build, - * rather than silently leaving that subtype's children unwalked. + * Every non-`Var` node's children are walked via [PgNodeExpression.mapChildren], which is exhaustive + * over the sealed [PgNodeExpression] hierarchy, so a buried GROUP RTE `Var` — e.g. `count(*) + + * 0::bigint` when `0::bigint` is also the grouping key, sitting alongside an `Aggref` as an + * [PgNodeExpression.OpExpr] argument — cannot silently escape substitution. * * @param depth remaining recursion budget, mirroring [MAX_EXPRESSION_DEPTH]; once exhausted, * [expression] is returned unchanged — failing toward the current, un-substituted (and therefore @@ -75,61 +57,13 @@ internal fun substituteGroupRteVars( ): PgNodeExpression { if (depth <= 0) return expression val recurse = { child: PgNodeExpression -> substituteGroupRteVars(child, groupExpressionsByVarno, depth - 1) } - return when (expression) { - is PgNodeExpression.Var -> { - val resolved = if (expression.levelsUp == 0) { - groupExpressionsByVarno[expression.varno]?.let { groupExpressions -> - groupExpressions.getOrNull(expression.varattno - 1) - } - } else { - null - } - if (resolved != null && resolved !is PgNodeExpression.Unknown) resolved else expression + if (expression !is PgNodeExpression.Var) return expression.mapChildren(recurse) + val resolved = if (expression.levelsUp == 0) { + groupExpressionsByVarno[expression.varno]?.let { groupExpressions -> + groupExpressions.getOrNull(expression.varattno - 1) } - - is PgNodeExpression.Const, - is PgNodeExpression.SqlValueFunction, - is PgNodeExpression.NextValExpr, - is PgNodeExpression.Unknown, - -> expression - - is PgNodeExpression.FuncExpr -> expression.copy(arguments = expression.arguments.map(recurse)) - is PgNodeExpression.OpExpr -> expression.copy(arguments = expression.arguments.map(recurse)) - is PgNodeExpression.ScalarArrayOpExpr -> expression.copy(arguments = expression.arguments.map(recurse)) - is PgNodeExpression.CoalesceExpr -> expression.copy(arguments = expression.arguments.map(recurse)) - is PgNodeExpression.NullIfExpr -> expression.copy(arguments = expression.arguments.map(recurse)) - is PgNodeExpression.MinMaxExpr -> expression.copy(arguments = expression.arguments.map(recurse)) - is PgNodeExpression.Aggref -> expression.copy(arguments = expression.arguments.map(recurse)) - is PgNodeExpression.WindowFunc -> expression.copy(arguments = expression.arguments.map(recurse)) - is PgNodeExpression.SubLink -> expression.copy(outerOperand = expression.outerOperand?.let(recurse)) - is PgNodeExpression.CaseExpr -> expression.copy( - resultExpressions = expression.resultExpressions.map(recurse), - defaultResult = expression.defaultResult?.let(recurse), - testExpression = expression.testExpression?.let(recurse), - whenConditions = expression.whenConditions.map(recurse), - ) - - is PgNodeExpression.BoolExpr -> expression.copy(arguments = expression.arguments.map(recurse)) - is PgNodeExpression.RelabelType -> expression.copy(argument = recurse(expression.argument)) - is PgNodeExpression.CoerceViaIo -> expression.copy(argument = recurse(expression.argument)) - is PgNodeExpression.ArrayCoerceExpr -> expression.copy(argument = recurse(expression.argument)) - is PgNodeExpression.CollateExpr -> expression.copy(argument = recurse(expression.argument)) - is PgNodeExpression.CoerceToDomain -> expression.copy(argument = recurse(expression.argument)) - is PgNodeExpression.NullTest -> expression.copy(argument = recurse(expression.argument)) - is PgNodeExpression.BooleanTest -> expression.copy(argument = recurse(expression.argument)) - is PgNodeExpression.DistinctExpr -> expression.copy(arguments = expression.arguments.map(recurse)) - is PgNodeExpression.ArrayExpr -> expression.copy(elements = expression.elements.map(recurse)) - is PgNodeExpression.RowExpr -> expression.copy(arguments = expression.arguments.map(recurse)) - is PgNodeExpression.GroupingFunc -> expression.copy(arguments = expression.arguments.map(recurse)) - is PgNodeExpression.FieldSelect -> expression.copy(argument = recurse(expression.argument)) - is PgNodeExpression.JsonIsPredicate -> expression.copy(argument = recurse(expression.argument)) - is PgNodeExpression.JsonConstructorExpr -> expression.copy(arguments = expression.arguments.map(recurse)) - is PgNodeExpression.JsonExpr -> expression.copy( - argument = recurse(expression.argument), - onEmptyDefault = expression.onEmptyDefault?.let(recurse), - onErrorDefault = expression.onErrorDefault?.let(recurse), - ) - - is PgNodeExpression.XmlExpr -> expression.copy(arguments = expression.arguments.map(recurse)) + } else { + null } + return if (resolved != null && resolved !is PgNodeExpression.Unknown) resolved else expression } diff --git a/generator/src/main/kotlin/norm/generator/ImplementationBuilder.kt b/generator/src/main/kotlin/norm/generator/ImplementationBuilder.kt index ca6702ed..0148aead 100644 --- a/generator/src/main/kotlin/norm/generator/ImplementationBuilder.kt +++ b/generator/src/main/kotlin/norm/generator/ImplementationBuilder.kt @@ -115,28 +115,12 @@ private fun TypeSpec.Builder.addManyImplementation(statement: SqlStatement) { val mapperReturnType = resultRowShape.mapperReturnType val returnTypeVariable = TypeVariableName("Return") // 1. Private helper function - val helperFunction = FunSpec.builder(statement.name) + val helperFunction = mapperFunction(statement) .addModifiers(KModifier.PRIVATE) - .addTypeVariable(mapperReturnType) .addTypeVariable(returnTypeVariable) - .apply { - for ((index, parameter) in statement.parameters.withIndex()) { - addParameter(ParameterSpec(statement.getParameterName(index), statement.resolveColumnType(parameter.column!!))) - } - } - .addParameter( - ParameterSpec( - MAPPER_PARAMETER_NAME, - LambdaTypeName.get( - parameters = resultRowShape.creationParameters.toTypedArray(), - returnType = mapperReturnType, - ), - ), - ) .addParameter( "processor", - ClassName("norm", "ManyProcessor") - .parameterizedBy(mapperReturnType, returnTypeVariable), + MANY_PROCESSOR.parameterizedBy(mapperReturnType, returnTypeVariable), ) .returns(returnTypeVariable) .addStatement("val sql = %S", statement.sql) @@ -160,24 +144,8 @@ private fun TypeSpec.Builder.addManyImplementation(statement: SqlStatement) { .build() addFunction(helperFunction) // 2. Public Many variant: override fun queryName(mapper: ...) -> Many - val manyFunction = FunSpec.builder(statement.name) + val manyFunction = mapperFunction(statement) .addModifiers(KModifier.OVERRIDE) - .addTypeVariable(mapperReturnType) - .apply { - for ((index, parameter) in statement.parameters.withIndex()) { - addParameter(ParameterSpec(statement.getParameterName(index), statement.resolveColumnType(parameter.column!!))) - } - } - .addParameter( - ParameterSpec( - MAPPER_PARAMETER_NAME, - LambdaTypeName.get( - parameters = resultRowShape.creationParameters.toTypedArray(), - returnType = mapperReturnType, - ), - ), - ) - .returns(statement.command.applyTo(mapperReturnType)) .apply { val args = ( statement.parameters.indices.map { CodeBlock.of("%N", statement.getParameterName(it)) } + listOf( @@ -191,19 +159,9 @@ private fun TypeSpec.Builder.addManyImplementation(statement: SqlStatement) { addFunction(manyFunction) // 3. If eligible, public Query variant: override fun queryNameDynamically(mapper: ...) -> Query if (statement.canBeDynamic) { - val dynamicName = "${statement.name}Dynamically" - val dynamicFunction = FunSpec.builder(dynamicName) + val dynamicFunction = mapperFunction(statement).build() + .toBuilder("${statement.name}Dynamically") .addModifiers(KModifier.OVERRIDE) - .addTypeVariable(mapperReturnType) - .addParameter( - ParameterSpec( - MAPPER_PARAMETER_NAME, - LambdaTypeName.get( - parameters = resultRowShape.creationParameters.toTypedArray(), - returnType = mapperReturnType, - ), - ), - ) .returns(Command.NORM_QUERY.parameterizedBy(mapperReturnType)) .addStatement( "return %N(%N) { sql, rowReader, _ -> driver.dynamic(sql, rowReader) }", @@ -484,6 +442,8 @@ private fun buildBatchWithReturn(statement: SqlStatement): FunSpec = batchWithRe endControlFlow() }.build() +private val MANY_PROCESSOR = ClassName(RUNTIME_PACKAGE, "ManyProcessor") + private val PROCESS_EXEC_RESULTS = MemberName(RUNTIME_PACKAGE, "combineExecBatchResults") private val READ_GENERATED_KEYS = MemberName(RUNTIME_PACKAGE, "readGeneratedKeys") diff --git a/generator/src/main/kotlin/norm/generator/JdbcAnalyzer.kt b/generator/src/main/kotlin/norm/generator/JdbcAnalyzer.kt index 2d27e787..257bfc5c 100644 --- a/generator/src/main/kotlin/norm/generator/JdbcAnalyzer.kt +++ b/generator/src/main/kotlin/norm/generator/JdbcAnalyzer.kt @@ -7,8 +7,9 @@ import java.sql.ResultSetMetaData /** * Analyzes PostgreSQL schemas and queries using JDBC metadata APIs. * - * Produces the same Wire protobuf types ([Catalog], [Query]) that the generator consumes, - * replacing the previous sqlc-based pipeline with direct database introspection. + * Produces the [Catalog] and [Query] model data classes (see + * `generator/src/main/kotlin/norm/generator/Model.kt`) that the generator consumes, + * using direct database introspection. * * Uses [DatabaseMetaData] for schema introspection and * [java.sql.PreparedStatement.getMetaData] / [java.sql.PreparedStatement.getParameterMetaData] @@ -45,7 +46,6 @@ public class JdbcAnalyzer(private val connection: Connection) { } return Catalog( - defaultSchema = schemas.first(), schemas = schemaObjects, ) } @@ -58,7 +58,7 @@ public class JdbcAnalyzer(private val connection: Connection) { * * @param parsedQuery The query parsed from a SQL file. * @param catalog The schema catalog, used to attach table references to result columns. - * @return A [Query] proto object with full type information. + * @return A [Query] with full type information. */ public fun analyzeQuery(parsedQuery: ParsedQuery, catalog: Catalog): Query { val jdbcSql = parsedQuery.sql @@ -75,7 +75,7 @@ public class JdbcAnalyzer(private val connection: Connection) { if (isCallStatement) { // CALL statements don't return result sets and may not support getMetaData() resultColumns = emptyList() - parameters = analyzeCallParameters(parsedQuery.sql, jdbcSql) + parameters = analyzeCallParameters(parsedQuery.sql, jdbcSql, catalog) } else { connection.prepareStatement(jdbcSql).use { ps -> resultColumns = buildResultColumns(ps.metaData, catalog, parsedQuery.sql) @@ -266,10 +266,10 @@ public class JdbcAnalyzer(private val connection: Connection) { */ private fun buildParameters( pmd: java.sql.ParameterMetaData, - inferredNames: Map = emptyMap(), - notNullByParameter: Map = emptyMap(), - inferredParameters: Map = emptyMap(), - catalog: Catalog? = null, + inferredNames: Map, + notNullByParameter: Map, + inferredParameters: Map, + catalog: Catalog, ): List { val parameters = mutableListOf() for (i in 1..pmd.parameterCount) { @@ -280,7 +280,7 @@ public class JdbcAnalyzer(private val connection: Connection) { val columnName = inferred?.columnName ?: inferred?.name // Look up the catalog column once for type name and comment resolution. - val catalogColumn = if (catalog != null && tableName != null && columnName != null) { + val catalogColumn = if (tableName != null && columnName != null) { catalog.findColumn(tableName, columnName) } else { null @@ -311,7 +311,7 @@ public class JdbcAnalyzer(private val connection: Connection) { arrayDims = if (isArray) 1 else 0, comment = comment, type = Identifier(name = typeName), - table = if (catalog != null && tableName != null) resolveTableIdentifier(tableName, catalog) else null, + table = tableName?.let { resolveTableIdentifier(it, catalog) }, originalName = columnName.orEmpty(), ), ), @@ -326,7 +326,7 @@ public class JdbcAnalyzer(private val connection: Connection) { * Prefers `pg_proc` lookup for argument names since JDBC `ParameterMetaData` only provides types. * Falls back to preparing the statement directly if the procedure isn't found in `pg_proc`. */ - private fun analyzeCallParameters(originalSql: String, jdbcSql: String): List { + private fun analyzeCallParameters(originalSql: String, jdbcSql: String, catalog: Catalog): List { val procName = CALL_PROCEDURE_NAME.find(originalSql)?.groupValues?.get(1) if (procName != null) { val params = catalogLoader.lookupProcedureParameters(procName) @@ -335,7 +335,7 @@ public class JdbcAnalyzer(private val connection: Connection) { return try { connection.prepareStatement(jdbcSql).use { ps -> - buildParameters(ps.parameterMetaData) + buildParameters(ps.parameterMetaData, emptyMap(), emptyMap(), emptyMap(), catalog) } } catch (_: Exception) { emptyList() @@ -393,19 +393,10 @@ public class JdbcAnalyzer(private val connection: Connection) { */ public fun buildIdentifierQuoter(): (String) -> String { val reservedWords = fetchReservedWords() - return { identifier -> - // PostgreSQL doubles every embedded double quote in a quoted identifier, the same way a quoted - // string literal doubles an embedded single quote -- without it, a column literally named a"b - // wraps unmodified into "a"b", read as "a" followed by an unterminated b" token. - if (needsQuoting(identifier, reservedWords)) "\"${identifier.replace("\"", "\"\"")}\"" else identifier - } + return { identifier -> quoteSqlIdentifierIfNeeded(identifier, reservedWords) } } private companion object { private val CALL_PROCEDURE_NAME = Regex("""CALL\s+(\w+)\s*\(""", RegexOption.IGNORE_CASE) - private val SAFE_IDENTIFIER = Regex("[a-z_][a-z0-9_\$]*") - - private fun needsQuoting(identifier: String, reservedWords: Set): Boolean = - identifier.lowercase() in reservedWords || !identifier.matches(SAFE_IDENTIFIER) } } diff --git a/generator/src/main/kotlin/norm/generator/KdocRendering.kt b/generator/src/main/kotlin/norm/generator/KdocRendering.kt new file mode 100644 index 00000000..db676140 --- /dev/null +++ b/generator/src/main/kotlin/norm/generator/KdocRendering.kt @@ -0,0 +1,278 @@ +// PropertySource is the only top-level class in this file, but the file also holds every free +// function that renders it into KDoc (addClassKdoc and its helpers) — deliberately, per this +// file's role as the single home for KDoc-rendering logic, not a naming slip. +@file:Suppress("MatchingDeclarationName") + +package norm.generator + +import com.squareup.kotlinpoet.ANY +import com.squareup.kotlinpoet.PropertySpec +import com.squareup.kotlinpoet.TypeSpec + +/** + * Describes a property's source in the database. + * + * @property propertyName The Kotlin property name. + * @property comment The Postgres column comment. Empty if none. + * @property sourceTable The database table the column comes from. `null` for computed columns. + * @property sourceColumn The original column name in the database. `null` for computed columns. + * @property expression The SQL expression for computed columns (e.g. `COUNT(*)`). Empty if not computed. + */ +internal data class PropertySource( + val propertyName: String, + val comment: String, + val sourceTable: String?, + val sourceColumn: String?, + val expression: String = "", +) + +/** + * Adds a class-level KDoc block with an optional description, table mapping, and `@property` tags. + * + * Produces a single consolidated KDoc block rather than separate per-property doc comments, which is the + * idiomatic Kotlin style for data classes with constructor properties. + * + * For table projections, the table name is shown as "Maps to the `X` table.". + * For query projections, the SQL is included and source columns are shown per-property as `table.column` references. + * + * @param classComment The table or class-level comment. May be empty. + * @param tableName The database table this class fully maps to. `null` for ad-hoc query projections. + * @param properties Source information for each property. + * @param sql The SQL query text. Included in query projection KDoc as a fenced code block. + * @param reservedWords The connected PostgreSQL server's reserved keywords, forwarded to + * [quoteSqlIdentifierIfNeeded] for each property's `` `table.column` `` source reference. Empty + * for a table projection, which never renders a source reference at all. + */ +internal fun TypeSpec.Builder.addClassKdoc( + classComment: String, + tableName: String?, + properties: List, + sql: String = "", + reservedWords: Set = emptySet(), +) { + val hasTableMapping = tableName != null + // KotlinPoet's KDoc emission rewrites "/*"/"*/" to "/*"/"*/" inside every KDoc block so a + // literal block comment can't prematurely close the surrounding "/** ... */" comment, but CommonMark + // never decodes that HTML entity back inside a fenced code block -- see + // containsUnescapableBlockCommentDelimiter. Declining the whole fenced block is the only correct + // choice for a query containing either sequence. + val canRenderSqlVerbatim = sql.isNotEmpty() && !containsUnescapableBlockCommentDelimiter(sql) + // A property whose name can't be rendered as a `@property` name token at all + // (formatAsKdocPropertyReference returns `null`) is dropped here rather than emitted with a + // mangled name that reads back as a different property than the one actually declared. + val documentedProperties = properties.mapNotNull { property -> + if (!property.hasDocumentation(hasTableMapping, reservedWords)) return@mapNotNull null + val formattedName = property.propertyName.formatAsKdocPropertyReference() ?: return@mapNotNull null + formattedName to property + } + if (classComment.isEmpty() && !hasTableMapping && !canRenderSqlVerbatim && documentedProperties.isEmpty()) return + + val kdoc = buildString { + if (classComment.isNotEmpty()) { + append(classComment) + } + if (hasTableMapping) { + if (isNotEmpty()) append("\n\n") + append("Maps to the `$tableName` table.") + } + if (canRenderSqlVerbatim) { + if (isNotEmpty()) append("\n\n") + // A fixed 3-backtick fence breaks if sql itself contains a run of 3+ backticks -- a line + // matching or exceeding the fence's own length terminates the fenced block early. A fence one + // backtick longer than any run already in sql can never be mistaken for a closing fence. + val fence = markdownFenceDelimiter(sql) + append(fence).append("sql\n") + append(sql.trim()) + append("\n").append(fence) + } + if (documentedProperties.isNotEmpty()) { + if (isNotEmpty()) append("\n\n") + for ((index, formattedNameAndProperty) in documentedProperties.withIndex()) { + val (formattedName, property) = formattedNameAndProperty + append("@property $formattedName ") + if (property.comment.isNotEmpty()) { + // Every `@property` line shares one CommonMark paragraph (no blank line between them), so + // an unescaped backtick in one comment could pair with a later property's own + // source-reference span instead of closing here. Escaping it keeps it from ever being read + // as a code-span delimiter. + append(escapeMarkdownBacktick(property.comment)) + } + if (!hasTableMapping) { + val source = property.sourceReference(reservedWords) + if (source != null) { + if (property.comment.isNotEmpty()) append(" ") + append("($source)") + } + } + if (index < documentedProperties.lastIndex) append("\n") + } + } + } + addKdoc("%L", kdoc) +} + +/** + * Whether this property has any documentation to show in KDoc. + */ +private fun PropertySource.hasDocumentation(hasTableMapping: Boolean, reservedWords: Set): Boolean = + comment.isNotEmpty() || (!hasTableMapping && sourceReference(reservedWords) != null) + +/** + * Whether KotlinPoet would backtick-quote the property declaration it renders for [name]. + * + * KotlinPoet escapes a declaration name for four independent reasons -- not a legal Java identifier, + * one of its own reserved `KEYWORDS`, contains `$`, or is all underscores -- and both the rule and + * the keyword set are `internal` to it, so a copy here would drift. Rendering a throwaway + * [PropertySpec] asks KotlinPoet directly instead. + * + * Tests for a backtick anywhere in the rendered text rather than for `` `$name` `` specifically: + * KotlinPoet's line wrapper substitutes a space for the characters it reserves as wrapping markers + * (U+00B7 and U+2662), so such a name is escaped in the output without appearing there verbatim. + * Callers must rule out a name containing its own backtick first -- KotlinPoet treats one as already + * escaped and skips all four checks. + */ +private fun needsKotlinPoetDeclarationBackticks(name: String): Boolean = + PropertySpec.builder(name, ANY).build().toString().contains('`') + +/** + * Formats a Kotlin property name for use as the name token in a KDoc `@property` tag. + * + * KDoc's `@property` tag takes exactly one name token before the description text begins, so a + * property name containing a space or other non-identifier character (e.g. the Kotlin property + * `` `My Col` `` generated for a quoted SQL column `"My Col"`) must be wrapped in backticks here too + * -- otherwise `@property My Col Some comment.` reads as a property literally named `My`. The name is + * left bare only when the declaration KotlinPoet renders for it is bare too, so the two never + * disagree. + * + * Uses [wrapInBacktickDelimiter]'s longest-run rule rather than a fixed single-backtick wrap, since a + * name containing its own literal backtick (e.g. `` a`b ``) would otherwise close the `@property` + * tag's span early, corrupting the rest of the line. + * + * Returns `null` — decline, emit no `@property` line at all — when [this] contains a literal + * block-comment open or close delimiter ([containsUnescapableBlockCommentDelimiter]): widening the + * backtick delimiter fixes the span, but KotlinPoet's KDoc emission still rewrites the delimiter + * itself to an HTML entity, which would render a tag naming a different property than the one + * actually declared. + * + * This fixes only the KDoc span; it does not and cannot fix the Kotlin property declaration itself + * (`` public val `a\`b`: ... ``), which is not valid Kotlin — a backtick-quoted identifier cannot + * contain a backtick, and there is no escape for one. That is a separate, pre-existing defect in how + * a column's raw database identifier becomes a Kotlin property name, left unfixed here because the + * same field also carries the identifier back into generated SQL and catalog lookups. + */ +private fun String.formatAsKdocPropertyReference(): String? = when { + !contains('`') && !needsKotlinPoetDeclarationBackticks(this) -> this + containsUnescapableBlockCommentDelimiter(this) -> null + else -> wrapInBacktickDelimiter(this) +} + +/** + * Returns a source reference string for display in KDoc, or `null` if none is available (either + * there is nothing to reference, or [markdownInlineCodeSpan] could not render it faithfully — see + * that function's own KDoc for when that happens). + * + * - For columns from a table: `` `table."Column"` `` — each identifier individually quoted via + * [quoteSqlIdentifierIfNeeded] exactly as PostgreSQL requires it written back into SQL, so this + * can always be pasted into a query verbatim. + * - For computed expressions: `` `COUNT(*)` `` + * + * @param reservedWords The connected server's reserved keywords, forwarded to + * [quoteSqlIdentifierIfNeeded]. + */ +private fun PropertySource.sourceReference(reservedWords: Set): String? = when { + sourceTable != null -> { + val qualifiedColumn = sourceColumn?.let { quoteSqlIdentifierIfNeeded(it, reservedWords) }.orEmpty() + markdownInlineCodeSpan("${quoteSqlIdentifierIfNeeded(sourceTable, reservedWords)}.$qualifiedColumn") + } + expression.isNotEmpty() -> markdownInlineCodeSpan(expression) + else -> null +} + +/** + * The longest run of consecutive backtick characters anywhere in [text], or `0` if [text] contains + * none. Used by [markdownInlineCodeSpan] and [markdownFenceDelimiter] to pick a delimiter that can + * never be mistaken for a same-length run already inside [text]. + */ +private fun longestBacktickRun(text: String): Int { + var longest = 0 + var current = 0 + for (character in text) { + if (character == '`') { + current++ + if (current > longest) longest = current + } else { + current = 0 + } + } + return longest +} + +/** + * Wraps [text] in a Markdown inline code span that renders back to exactly [text], or `null` if no + * inline code span can carry it faithfully. + * + * Two hazards: + * - A run of backticks inside [text] as long as the span's own delimiter would be read as the + * closing delimiter, ending the span early. Fixed by using a delimiter one backtick longer than + * [text]'s own longest run ([longestBacktickRun]), with a padding space on each side when [text] + * itself starts or ends with a backtick. + * - A raw newline inside [text] (e.g. a string literal containing one). CommonMark folds a line + * ending inside an inline code span to a single space when rendering, silently changing the value. + * No delimiter choice can fix this — the corruption happens during rendering — so this declines. + * - A literal block-comment open or close delimiter inside [text] — see + * [containsUnescapableBlockCommentDelimiter] for why that is a third, un-fixable-by-delimiter + * hazard. + */ +internal fun markdownInlineCodeSpan(text: String): String? { + if (text.contains('\n') || text.contains('\r')) return null + if (containsUnescapableBlockCommentDelimiter(text)) return null + return wrapInBacktickDelimiter(text) +} + +/** + * Whether [text] contains `/*` or `*/`. KotlinPoet's KDoc emission unconditionally rewrites either to + * `/*`/`*/` so a literal block comment can never prematurely close the surrounding KDoc + * comment, but CommonMark never decodes that HTML entity back inside an inline code span or fenced + * code block — the two constructs [markdownInlineCodeSpan] and [TypeSpec.Builder.addClassKdoc]'s + * `sql` block render as. Once that rewrite happens there is no delimiter choice that can carry [text] + * back to its own value, so both callers decline instead. + */ +internal fun containsUnescapableBlockCommentDelimiter(text: String): Boolean = + text.contains("/*") || text.contains("*/") + +/** + * Backslash-escapes every literal backtick in [text] so it can never be read as a CommonMark + * inline-code-span delimiter, without altering the character [text] renders as. + * + * [TypeSpec.Builder.addClassKdoc] appends every property's [PropertySource.comment] into one + * continuous CommonMark paragraph shared by every `@property` line, so an unescaped backtick in one + * comment could pair with a backtick belonging to a later property's own source-reference span, + * corrupting every span in between. Escaping here removes the character from delimiter-matching + * entirely. + * + * Escapes [text]'s own literal backslashes first, before escaping backticks: escaping only the + * backtick is defeated when [text] already has a backslash immediately before one (e.g. + * `` 'weird \`' ``) — the naive replacement produces `` \\` ``, which CommonMark reads as an escaped + * backslash followed by an unescaped, still-open backtick. + */ +internal fun escapeMarkdownBacktick(text: String): String = text.replace("\\", "\\\\").replace("`", "\\`") + +/** + * Wraps [text] in a backtick-delimited span using a delimiter one backtick longer than [text]'s own + * longest internal run ([longestBacktickRun]), with a padding space on each side when [text] starts + * or ends with a backtick. Shared by [markdownInlineCodeSpan] and [formatAsKdocPropertyReference], + * whose spans are both subject to the same backtick-collision hazard. + */ +private fun wrapInBacktickDelimiter(text: String): String { + val delimiter = "`".repeat(longestBacktickRun(text) + 1) + val needsPadding = text.startsWith("`") || text.endsWith("`") + return if (needsPadding) "$delimiter $text $delimiter" else "$delimiter$text$delimiter" +} + +/** + * A backtick-fence delimiter (` ``` `, or longer) that can open a Markdown fenced code block + * containing [text] without [text] itself supplying a same-length or longer backtick run that + * CommonMark would read as the block's own closing fence — one backtick longer than [text]'s own + * longest run ([longestBacktickRun]), never shorter than the conventional 3. + */ +internal fun markdownFenceDelimiter(text: String): String = "`".repeat(maxOf(3, longestBacktickRun(text) + 1)) diff --git a/generator/src/main/kotlin/norm/generator/Main.kt b/generator/src/main/kotlin/norm/generator/Main.kt index dab63688..c529b681 100644 --- a/generator/src/main/kotlin/norm/generator/Main.kt +++ b/generator/src/main/kotlin/norm/generator/Main.kt @@ -43,9 +43,9 @@ public fun generateCode( reservedWords: Set, typeMappings: List = emptyList(), ): List { - val generator = TypeRepository(packageName, catalog, typeMappings, reservedWords) + val typeRepository = TypeRepository(packageName, catalog, typeMappings, reservedWords) - val resolvedQueries = queries.map { SqlStatement(catalog, it, generator) } + val resolvedQueries = queries.map { SqlStatement(catalog, it, typeRepository) } val queriesInterface = ClassName(packageName, "Queries") val interfaceCode = generateQueryInterface(resolvedQueries, "Queries", frameworks) @@ -53,8 +53,7 @@ public fun generateCode( val typeOverridePostgresTypes = typeMappings.filter { it.isTypeLevel }.map { it.postgresType }.toSet() // Build enum + adapter TypeSpecs for all enums discovered during query resolution. - // discoveredEnums is populated as a side effect of resolving column types above. - val enumTypeSpecs = generator.discoveredEnums + val enumTypeSpecs = typeRepository.discoveredEnums .filter { it.name !in typeOverridePostgresTypes } .sortedBy { it.name } .flatMap { enumDefinition -> @@ -65,7 +64,7 @@ public fun generateCode( } // Build value class + adapter TypeSpecs for all domains discovered during query resolution. - val domainTypeSpecs = generator.discoveredDomains + val domainTypeSpecs = typeRepository.discoveredDomains .filter { it.name !in typeOverridePostgresTypes } .sortedBy { it.name } .flatMap { domain -> @@ -75,25 +74,19 @@ public fun generateCode( ) } - val classCode = - generateQueryImplementation( - resolvedQueries, - queriesInterface, - frameworks, - generator.discoveredEnums, - generator.discoveredDomains, - packageName, - typeMappings, - typeOverridePostgresTypes, - catalog, - ) + // Computed only now, after generateQueryInterface has resolved every query parameter's column + // type — see adapterParameters' KDoc for why this ordering matters. + val adapterParameters = + adapterParameters(typeRepository, typeMappings, catalog, packageName, typeOverridePostgresTypes) + + val classCode = generateQueryImplementation(resolvedQueries, queriesInterface, frameworks, adapterParameters) val connectionProviders = generateConnectionProviders(packageName, frameworks) val typeSpecFiles = ( sequenceOf( interfaceCode, classCode, - ) + generator.requiredTypes + enumTypeSpecs + domainTypeSpecs + ) + typeRepository.requiredTypes + enumTypeSpecs + domainTypeSpecs ).map { val fileSpec = FileSpec.builder(packageName, "${it.name}.kt") .addType(it) @@ -105,40 +98,47 @@ public fun generateCode( return typeSpecFiles + connectionProviders } -private fun generateQueryImplementation( - queries: List, - interfaceType: ClassName, - frameworks: Set, - discoveredEnums: Set, - discoveredDomains: Set, - packageName: String, +/** + * A single adapter constructor parameter for the generated `PostgresQueries` implementation. + * + * @param propertyName The constructor parameter (and private property) name. + * @param adapterType The `ColumnAdapter` type of the parameter. + * @param defaultClass The adapter class to instantiate as the parameter's default value + * (`= DefaultClass()`), or `null` for user-configured adapters, which have no default and must be + * supplied explicitly. + */ +private data class AdapterParameter(val propertyName: String, val adapterType: TypeName, val defaultClass: ClassName?) + +/** + * Computes the adapter constructor parameters for the generated `PostgresQueries` implementation. + * + * Adapter parameters come in two groups: + * 1. User-configured adapters (no default) — must come first in the constructor. + * 2. Auto-generated adapters, for enums and domains discovered while resolving column types (with a + * default) — come after. + * + * Must be called only after every query has been resolved into a Kotlin interface (i.e., after + * [generateQueryInterface]). [TypeRepository.discoveredEnums] and [TypeRepository.discoveredDomains] + * are populated as a side effect of resolving column types, and a query *parameter*'s column type is + * first resolved while building the interface method for that query, not while constructing + * [SqlStatement]. Calling this before every query is resolved silently drops any enum or domain + * referenced only as a query parameter. + * + * @param typeOverridePostgresTypes Postgres type names with a user-configured type-level override, + * already computed by the caller so it's derived from [typeMappings] exactly once. + */ +private fun adapterParameters( + typeRepository: TypeRepository, typeMappings: List, - typeOverridePostgresTypes: Set, catalog: Catalog, -): TypeSpec { - val constructorBuilder = FunSpec.constructorBuilder() - .addParameter("connectionProvider", CONNECTION_PROVIDER) - - val classBuilder = TypeSpec.classBuilder("PostgresQueries") - .addSuperinterface(interfaceType) - - if (usesNormManagedTransactions(frameworks)) { - // Norm-managed transactions: the concrete PostgresQueries extends RealTransactable so callers can - // run transaction { } directly. Data-integration frameworks instead delegate to @Transactional. - classBuilder.superclass(REAL_TRANSACTABLE) - classBuilder.addSuperclassConstructorParameter("connectionProvider") - } - - // Adapter parameters come in two groups: - // 1. User-configured adapters (no default) — must come first in the constructor - // 2. Auto-generated adapters (with default) — come after - data class AdapterParam(val propertyName: String, val adapterType: TypeName, val defaultClass: ClassName?) - + packageName: String, + typeOverridePostgresTypes: Set, +): List { // User-configured adapter params (no default value → must come first) val userAdapterParams = typeMappings.map { mapping -> val applicationTypeName = parseTypeName(mapping.kotlinType) val databaseTypeName = resolveWireTypeName(mapping, catalog) - AdapterParam( + AdapterParameter( userAdapterPropertyName(mapping), COLUMN_ADAPTER.parameterizedBy(applicationTypeName, databaseTypeName), null, @@ -147,23 +147,23 @@ private fun generateQueryImplementation( // Auto-generated adapter params (with default → come after) val autoAdapterParams = buildList { - for (enumDefinition in discoveredEnums) { + for (enumDefinition in typeRepository.discoveredEnums) { if (enumDefinition.name in typeOverridePostgresTypes) continue val enumClassName = ClassName(packageName, enumDefinition.name.snakeToCamelCase().titleCase()) add( - AdapterParam( + AdapterParameter( adapterPropertyName(enumDefinition), COLUMN_ADAPTER.parameterizedBy(enumClassName, String::class.asTypeName()), adapterClassName(enumDefinition, packageName), ), ) } - for (domain in discoveredDomains) { + for (domain in typeRepository.discoveredDomains) { if (domain.name in typeOverridePostgresTypes) continue val valueClassName = domainValueClassName(domain, packageName) val baseKotlinType = domainKotlinBaseType(domain.baseType) add( - AdapterParam( + AdapterParameter( domainAdapterPropertyName(domain), COLUMN_ADAPTER.parameterizedBy(valueClassName, baseKotlinType), domainAdapterClassName(domain, packageName), @@ -172,7 +172,29 @@ private fun generateQueryImplementation( } }.sortedBy { it.propertyName } - for (param in userAdapterParams + autoAdapterParams) { + return userAdapterParams + autoAdapterParams +} + +private fun generateQueryImplementation( + queries: List, + interfaceType: ClassName, + frameworks: Set, + adapterParameters: List, +): TypeSpec { + val constructorBuilder = FunSpec.constructorBuilder() + .addParameter("connectionProvider", CONNECTION_PROVIDER) + + val classBuilder = TypeSpec.classBuilder("PostgresQueries") + .addSuperinterface(interfaceType) + + if (usesNormManagedTransactions(frameworks)) { + // Norm-managed transactions: the concrete PostgresQueries extends RealTransactable so callers can + // run transaction { } directly. Data-integration frameworks instead delegate to @Transactional. + classBuilder.superclass(REAL_TRANSACTABLE) + classBuilder.addSuperclassConstructorParameter("connectionProvider") + } + + for (param in adapterParameters) { val paramBuilder = ParameterSpec.builder(param.propertyName, param.adapterType) if (param.defaultClass != null) { paramBuilder.defaultValue("%T()", param.defaultClass) @@ -375,11 +397,11 @@ private fun resolveWireKotlinType(postgresType: String, catalog: Catalog): TypeN /** * Maps a Postgres base type name to the Kotlin type that JDBC delivers it as. * - * Delegates to [resolveJdbcTypeInfo] as the single source of truth for type mappings, so the set of + * Delegates to [resolveWireCodec] as the single source of truth for type mappings, so the set of * usable adapter wire types is the same as the set of usable domain bases ([domainKotlinBaseType]). * The two differ only in the error message they raise for a type with no entry. */ -private fun wireKotlinType(postgresType: String): TypeName = resolveJdbcTypeInfo(postgresType)?.kotlinType +private fun wireKotlinType(postgresType: String): TypeName = resolveWireCodec(postgresType)?.kotlinType ?: error( "Postgres type '$postgresType' cannot be used as the wire type for a custom adapter. " + "Supported wire types: text, varchar, bpchar, json, jsonb, int2, int4, int8, float4, float8, " + diff --git a/generator/src/main/kotlin/norm/generator/MergeSideNullability.kt b/generator/src/main/kotlin/norm/generator/MergeSideNullability.kt index 48149745..edbc4a92 100644 --- a/generator/src/main/kotlin/norm/generator/MergeSideNullability.kt +++ b/generator/src/main/kotlin/norm/generator/MergeSideNullability.kt @@ -17,7 +17,7 @@ internal data class MergeSideNullability(val targetCanBeAbsent: Boolean, val sou * `:mergeActionList`/text inspection. * * A `MERGE`'s match-optionality is invisible to `:varnullingrels` (see - * [PgCatalogLoader.mergeAbsentVarnos]): `WHEN NOT MATCHED BY SOURCE` and `WHEN NOT MATCHED [BY + * [ColumnNullabilityAnalyzer.mergeAbsentVarnos]): `WHEN NOT MATCHED BY SOURCE` and `WHEN NOT MATCHED [BY * TARGET] THEN INSERT` each mean one side of the underlying target/source comparison may have no * matching row, but PostgreSQL's `Var` nodes for either relation carry an empty nulling-relations set * regardless. The planner, however, executes that comparison as an ordinary join whose type encodes @@ -38,7 +38,7 @@ internal data class MergeSideNullability(val targetCanBeAbsent: Boolean, val sou * the plan — normally a single real table name, but a CTE source offers two candidates (its own * literal name, for a `MATERIALIZED` or otherwise non-inlined plan; and, when resolvable, the * single base table its body inlines to), since nothing in the parsed query tree says which shape - * the planner will choose (see [PgCatalogLoader.mergeAbsentVarnos]). At most one candidate can + * the planner will choose (see [ColumnNullabilityAnalyzer.mergeAbsentVarnos]). At most one candidate can * ever actually appear in a given plan, so offering more than one never risks attributing the * wrong side. * @return `null` when `EXPLAIN` fails, its JSON cannot be parsed, no plan node is uniquely diff --git a/generator/src/main/kotlin/norm/generator/Model.kt b/generator/src/main/kotlin/norm/generator/Model.kt index 1b0abdeb..346e844e 100644 --- a/generator/src/main/kotlin/norm/generator/Model.kt +++ b/generator/src/main/kotlin/norm/generator/Model.kt @@ -3,17 +3,9 @@ package norm.generator /** * A database catalog containing one or more schemas. * - * @property comment Catalog-level comment. Empty when absent. - * @property defaultSchema Name of the default schema (typically `"public"`). - * @property name Catalog name. Empty when not applicable. * @property schemas Schemas within this catalog. */ -public data class Catalog( - val comment: String = "", - val defaultSchema: String = "", - val name: String = "", - val schemas: List = emptyList(), -) { +public data class Catalog(val schemas: List = emptyList()) { /** * Finds a [Table] matching the given [Identifier] in the [Catalog]. @@ -57,30 +49,18 @@ public data class Catalog( /** * A database schema containing tables, enums, composite types, and domains. * - * @property comment Schema-level comment. Empty when absent. * @property name Schema name (e.g. `"public"`). * @property tables Tables and views in this schema. * @property enums User-defined enum types in this schema. - * @property compositeTypes User-defined composite types in this schema. * @property domains User-defined domain types in this schema. */ public data class Schema( - val comment: String = "", val name: String = "", val tables: List = emptyList(), val enums: List = emptyList(), - val compositeTypes: List = emptyList(), val domains: List = emptyList(), ) -/** - * A user-defined composite type. - * - * @property name Type name. - * @property comment Type-level comment. Empty when absent. - */ -public data class CompositeType(val name: String = "", val comment: String = "") - /** * A user-defined domain type (`CREATE DOMAIN name AS base_type CHECK (...)`). * Domains are thin wrappers over a base type with optional constraints. @@ -117,13 +97,12 @@ public data class Table( ) /** - * A three-part SQL identifier (catalog, schema, name). + * A two-part SQL identifier (schema, name). * - * @property catalog Catalog component. Empty when not applicable. * @property schema Schema component. Empty when unqualified. * @property name Object name. */ -public data class Identifier(val catalog: String = "", val schema: String = "", val name: String = "") +public data class Identifier(val schema: String = "", val name: String = "") /** * A column in a table or query result set. @@ -132,17 +111,10 @@ public data class Identifier(val catalog: String = "", val schema: String = "", * @property notNull `true` when the column has a `NOT NULL` constraint. * @property isArray `true` when the column is an array type. * @property comment Column-level comment. Empty when absent. - * @property length Column length (e.g. `VARCHAR(n)`). `0` when unspecified. - * @property isNamedParam `true` when this column represents a named parameter. - * @property isFuncCall `true` when this column originates from a function call. - * @property scope Scope qualifier for dotted references (e.g. `foo` in `foo.id`). * @property table Identifier of the table this column belongs to. `null` for computed columns. - * @property tableAlias Alias used for the table in the query. Empty when not aliased. * @property type Identifier of the column's data type. - * @property isSqlcSlice Sqlc-specific: `true` for slice parameters. * @property embedTable Sqlc-specific: table to embed. `null` when not embedding. * @property originalName Original column name before any aliasing. - * @property unsigned `true` for unsigned integer types. * @property arrayDims Number of array dimensions. `0` for non-array types. * @property isPrimaryKey `true` when this column is part of the primary key. * @property isAutoIncrement `true` when JDBC reports `IS_AUTOINCREMENT = "YES"`. @@ -165,17 +137,10 @@ public data class Column( val notNull: Boolean = false, val isArray: Boolean = false, val comment: String = "", - val length: Int = 0, - val isNamedParam: Boolean = false, - val isFuncCall: Boolean = false, - val scope: String = "", val table: Identifier? = null, - val tableAlias: String = "", val type: Identifier, - val isSqlcSlice: Boolean = false, val embedTable: Identifier? = null, val originalName: String = "", - val unsigned: Boolean = false, val arrayDims: Int = 0, val isPrimaryKey: Boolean = false, val isAutoIncrement: Boolean = false, @@ -201,7 +166,6 @@ public data class Column( * @property params Positional parameters for this query. * @property comments Comments associated with this query. * @property filename Source file containing this query. - * @property insertIntoTable Target table for INSERT queries. `null` for non-INSERT queries. * @property isSynthesizedInsert `true` when this query was synthesized by [CrudQuerySynthesizer]. * @property namedParameters Maps each 1-based JDBC parameter position to the named parameter that * produced it. Empty for queries using positional `?` parameters or synthesized CRUD queries. @@ -214,7 +178,6 @@ public data class Query( val params: List = emptyList(), val comments: List = emptyList(), val filename: String = "", - val insertIntoTable: Identifier? = null, val isSynthesizedInsert: Boolean = false, val namedParameters: Map = emptyMap(), ) diff --git a/generator/src/main/kotlin/norm/generator/NeverNullSafeLists.kt b/generator/src/main/kotlin/norm/generator/NeverNullSafeLists.kt index 6817b6f9..f4ea0cb1 100644 --- a/generator/src/main/kotlin/norm/generator/NeverNullSafeLists.kt +++ b/generator/src/main/kotlin/norm/generator/NeverNullSafeLists.kt @@ -4,7 +4,7 @@ package norm.generator * The `pg_catalog` function, cast, and operator signatures confirmed total on non-null input on * PostgreSQL 18 -- proven for every combination of non-null arguments, not merely "typical" ones, * including infinite, empty, or unbounded edge values. Backs - * [PgCatalogLoader.neverNullForNonNullInputOids] via [NEVER_NULL_FUNCTION_SIGNATURES], + * [NullabilityCatalog.neverNullForNonNullInputOids] via [NEVER_NULL_FUNCTION_SIGNATURES], * [NEVER_NULL_CAST_SIGNATURES], and [NEVER_NULL_OPERATOR_SIGNATURES] -- this data lives in its * own file, separate from [PgCatalogLoader]'s own job of loading catalog metadata. * diff --git a/generator/src/main/kotlin/norm/generator/NodeTreeNullabilityAnalyzer.kt b/generator/src/main/kotlin/norm/generator/NodeTreeNullabilityAnalyzer.kt index ea5f5093..a8d06f6c 100644 --- a/generator/src/main/kotlin/norm/generator/NodeTreeNullabilityAnalyzer.kt +++ b/generator/src/main/kotlin/norm/generator/NodeTreeNullabilityAnalyzer.kt @@ -21,75 +21,52 @@ import norm.generator.NodeTreeNullabilityAnalyzer.Companion.MAX_EXPRESSION_DEPTH * @param isOuterJoinNullable Returns `true` if the given `nullingRelations` set indicates the column * can be nulled by an outer join. Typically `true` when the set is non-empty. * @param isAlwaysNonNull Returns `true` for function OIDs that never return `null` for any - * combination of argument values, including when every argument is `null` (e.g., `concat`, which + * combination of argument values, including when every argument is `null` (e.g. `concat`, which * renders a `null` argument as an empty string) — but only for the ordinary (non-`VARIADIC`) - * calling form. `concat(VARIADIC arr)` is `null` when `arr` itself is `null` (PostgreSQL 16-18): - * `isNonNull`'s [PgNodeExpression.FuncExpr] branch checks - * [PgNodeExpression.FuncExpr.isVariadic] before trusting this callback at all, and this - * parameter's own guarantee never covers that form. See - * [PgCatalogLoader.alwaysNonNullFunctionOids]'s KDoc for why the non-`VARIADIC` guarantee must - * be unconditional in every argument position — `concat_ws` is deliberately not eligible here - * despite also being non-strict, because it depends on which argument is `null` (only a `null` - * separator, its first argument, makes the result `null`); see - * [isNonNullIffFirstArgumentNonNull] for how that case is modeled instead. + * calling form. `concat(VARIADIC arr)` is `null` when `arr` itself is `null` (PostgreSQL 16-18); + * `isNonNull`'s [PgNodeExpression.FuncExpr] branch checks [PgNodeExpression.FuncExpr.isVariadic] + * before trusting this callback for that form. * @param isNeverNullForNonNullInput Returns `true` for function/operator OIDs that are proven total * on non-null input — every combination of non-null arguments produces a non-null result (an * error is fine; only a silent `null` return disqualifies a candidate). `pg_proc.proisstrict` * alone cannot answer this: strict only guarantees NULL-in => NULL-out, never the converse, so - * this is required as an additional conjunct alongside [isStrict] below, never a substitute for - * it. See [PgCatalogLoader.neverNullForNonNullInputOids] for the safe-list this is normally - * backed by, and why omission from that list is always the safe default. That safe-list's - * verification (see `SafeListSweepTest`) covers only the ordinary, element-wise calling - * convention — `isNonNull`'s [PgNodeExpression.FuncExpr] branch never consults this - * parameter at all for a `VARIADIC` call (see [PgNodeExpression.FuncExpr.isVariadic]'s KDoc): - * the array argument being non-null says nothing about whether an element inside it is, and no - * function on the safe-list this backs is variadic today, so trusting it for that shape has - * never been verified. + * this must be checked alongside [isStrict], never as a substitute for it. Verified only for the + * ordinary, element-wise calling convention: `isNonNull`'s [PgNodeExpression.FuncExpr] branch + * never consults this parameter for a `VARIADIC` call, since the array argument being non-null + * says nothing about whether an element inside it is. * @param isLagLeadWithDefault Returns `true` for the 3-argument overloads of `lag` and `lead` window * functions, which return non-null when both the value and default arguments are non-null. * @param isFoldableToConst Returns `true` for function/operator OIDs that are IMMUTABLE and not - * set-returning (`pg_proc.provolatile = 'i' AND NOT proretset`). Used by - * [isSafeFromGroupingSetNullExtension]'s [foldsToConst] leg — see that method's KDoc. + * set-returning (`pg_proc.provolatile = 'i' AND NOT proretset`). * @param isNonNullIffFirstArgumentNonNull Returns `true` for function OIDs that are non-null if and * only if their first argument is non-null, regardless of any other argument's nullability, in * the ORDINARY (non-`VARIADIC`) calling form — used for [isNonNull] evaluation of - * [PgNodeExpression.FuncExpr] when [PgNodeExpression.FuncExpr.isVariadic] is `false`. Currently - * backs `concat_ws`: its first argument is the separator, and `concat_ws(null, 'x', 'y')` is - * `null` even though the later, individually-null-tolerant arguments are non-null (PostgreSQL - * 16-18). `concat_ws(',', VARIADIC arr)` is a different case this - * parameter's guarantee does not cover: it is `null` when `arr` itself is `null` even though the - * literal separator is non-null (also true on PostgreSQL 16-18). See - * [PgCatalogLoader.nonNullIffFirstArgumentNonNullFunctionOids] for the safe-list this is normally - * backed by, and why it is intentionally separate from [isAlwaysNonNull]. Also consulted by - * [isSafeFromGroupingSetNullExtension] for the identical non-`VARIADIC` `FuncExpr` shape. + * [PgNodeExpression.FuncExpr] when [PgNodeExpression.FuncExpr.isVariadic] is `false`. Backs + * `concat_ws`: its first argument is the separator, and `concat_ws(null, 'x', 'y')` is `null` + * even though the later, individually-null-tolerant arguments are non-null (PostgreSQL 16-18). + * `concat_ws(',', VARIADIC arr)` is a different case this parameter's guarantee does not cover: + * it is `null` when `arr` itself is `null` even though the literal separator is non-null (also + * true on PostgreSQL 16-18). * @param hasGroupingSets `true` when the query block this analyzer evaluates uses GROUPING SETS, * CUBE, or ROLLUP (see [PgNodeTreeParser.hasGroupingSets]). When `true`, [extractColumnNullability] * forces a result column nullable when it is itself a grouping key, or when its expression is not - * provably immune to the grouping-set null-extension mechanism — see [isSafeFromGroupingSetNullExtension] - * for the reasoning. Defaults to `false` (ordinary [isNonNull] evaluation only) for query blocks - * without grouping sets. + * provably immune to the grouping-set null-extension mechanism. Defaults to `false` (ordinary + * [isNonNull] evaluation only) for query blocks without grouping sets. * @param isSubLinkSubqueryColumnNotNull Returns `true` when [subselectBlock] — the raw `{QUERY ...}` * text of an `ANY_SUBLINK`'s or `ALL_SUBLINK`'s `:subselect` (see * [PgNodeExpression.SubLink.subselectBlock]) — produces exactly one non-junk output column and - * that column is provably non-null. Used by [isNonNull]'s `SubLink` branch as the third, most - * expensive leg of the identical `ANY_SUBLINK`/`ALL_SUBLINK` nullability rule (see that branch's - * own comment for the full three-condition rule and why each condition is required). Defaults to - * `{ false }` — every existing construction site and unit test that does not explicitly wire this - * callback stays conservative (nullable), which is also the correct behavior for a nested sublink - * once the caller's own depth budget for this analysis is exhausted (see - * `ColumnNullabilityAnalyzer`'s wiring of this callback for that budget). + * that column is provably non-null. Defaults to `{ false }`: every construction site that does + * not wire this callback stays conservative (nullable), which is also correct for a nested + * sublink once the caller's own depth budget for this analysis is exhausted. * @param forceNewNullable `true` when a `RETURNING WITH (OLD AS o, NEW AS n)` reference to `NEW` * (`Var.returningType == `[PgNodeExpression.VAR_RETURNING_TYPE_NEW]`) must be treated as - * unconditionally nullable, the same way [isNonNull]'s `Var` branch always treats `OLD` - * (`VAR_RETURNING_TYPE_OLD`) regardless of this flag. Set by the caller when the enclosing - * statement is a plain `DELETE` (`NEW` never exists — the row is gone; `NEW.col` - * is `NULL` for every row a `DELETE` returns) or a `MERGE` (an individual result row's `NEW` may - * or may not exist depending on which `WHEN` clause matched — e.g. `WHEN MATCHED THEN DELETE` - * leaves no `NEW` row — a fact this analyzer cannot isolate per-row any more than it can for an - * ordinary, non-`OLD`/`NEW` `MERGE` column; see [PgCatalogLoader.mergeAbsentVarnos]'s - * KDoc for that companion safety net). Left `false` (the default) for a plain `UPDATE`/`INSERT`, - * where the row a `RETURNING` clause reports on always has both an `OLD` and a `NEW` state, so - * `NEW` is exactly as trustworthy as an ordinary column reference. + * unconditionally nullable. Set by the caller when the enclosing statement is a plain `DELETE` + * (`NEW` never exists — the row is gone; `NEW.col` is `NULL` for every row a `DELETE` returns) or + * a `MERGE` (an individual result row's `NEW` may or may not exist depending on which `WHEN` + * clause matched — e.g. `WHEN MATCHED THEN DELETE` leaves no `NEW` row). Left `false` (the + * default) for a plain `UPDATE`/`INSERT`, where the row a `RETURNING` clause reports on always + * has both an `OLD` and a `NEW` state, so `NEW` is exactly as trustworthy as an ordinary column + * reference. */ internal class NodeTreeNullabilityAnalyzer( private val isStrict: (Int) -> Boolean, @@ -112,24 +89,17 @@ internal class NodeTreeNullabilityAnalyzer( * Extracts per-column nullability from a `pg_node_tree` text using full expression evaluation. * * Uses [PgNodeTreeParser] to parse the target list, then evaluates each non-junk entry with - * [isNonNull] for accurate expression-level nullability. Returns `true` (nullable) when - * `isNonNull` returns `false`. + * [isNonNull]. Returns `true` (nullable) when `isNonNull` returns `false`. * - * Before any of that, every target-list entry's expression is run through - * [substituteGroupRteVars] against [PgNodeTreeParser.parseGroupRteExpressions]'s result. On - * PostgreSQL 16 and 17 that map is always empty (no GROUP RTE exists), so this is a no-op and - * every entry's expression is exactly what [PgNodeTreeParser.parseTargetList] parsed. On - * PostgreSQL 18+, this restores the same tree shape 16/17 already have — the real grouping-key - * expression, not a `Var` referencing the synthesized `*GROUP*` RTE — so every rule below - * ([groupingSortGroupRefs], [groupingKeyExpressions], [isEffectivelyNonNull], - * [isSafeFromGroupingSetNullExtension], [isNonNull]) runs identically regardless of which - * PostgreSQL version produced [nodeTreeText]. This applies to a plain `GROUP BY` exactly as much - * as to `GROUPING SETS`/`CUBE`/`ROLLUP` — PostgreSQL 18 creates a GROUP RTE for a plain `GROUP BY` - * too — regardless of [hasGroupingSets]. + * Before that, every target-list entry's expression is run through [substituteGroupRteVars] + * against [groupExpressions]'s result. On PostgreSQL 16/17 that map is always empty (no GROUP + * RTE exists), so this is a no-op. On PostgreSQL 18+, a plain `GROUP BY` (not only `GROUPING + * SETS`/`CUBE`/`ROLLUP`) creates a synthesized `*GROUP*` RTE, and every target-list `Var` that + * references it is resolved back to the real grouping-key expression, restoring the same tree + * shape 16/17 produce directly. * - * CTE column resolution is handled by the caller through the [isSourceColumnNotNull] callback. - * The caller must include CTE column not-null information in this callback so that VAR nodes - * referencing CTE RTEs resolve correctly via the standard [isNonNull] Var evaluation path. + * The caller must fold CTE column not-null information into [isSourceColumnNotNull] so that `Var` + * nodes referencing CTE range-table entries resolve correctly. * * @param nodeTreeText the raw text value of `pg_rewrite.ev_action` * @return one `Boolean` per result column (in column order), where `true` means the column may @@ -138,7 +108,7 @@ internal class NodeTreeNullabilityAnalyzer( fun extractColumnNullability(nodeTreeText: String): List { val parsedEntries = parser.parseTargetList(nodeTreeText) if (parsedEntries.isEmpty()) return emptyList() - val groupRteExpressions = parser.parseGroupRteExpressions(nodeTreeText) + val groupRteExpressions = parser.parseRangeTableEntries(nodeTreeText).groupExpressions(parser) val entries = if (groupRteExpressions.isEmpty()) { parsedEntries } else { @@ -169,49 +139,35 @@ internal class NodeTreeNullabilityAnalyzer( * * When [hasGroupingSets] is `true`, [entry] is forced nullable when any of: * - [entry] is a grouping key itself: its [TargetEntry.sortGroupRef] is non-zero and appears in - * [groupingSortGroupRefs] (from [PgNodeTreeParser.parseGroupingSortGroupRefs]); or + * [groupingSortGroupRefs]. Alone this misses a *derived* expression over a key, e.g. + * `upper(lower(a))` when the key is `lower(a)` — caught by the third condition instead. * - [entry]'s expression structurally equals one of [groupingKeyExpressions] — a *duplicate* * occurrence of a grouping key expression that PostgreSQL did not assign the matching - * `ressortgroupref` to (see that parameter's KDoc for why this is a distinct case from the - * one above, not a redundant restatement of it); or - * - [entry]'s expression is not proven [isSafeFromGroupingSetNullExtension]. - * - * All three conditions are necessary and independent. The first alone would miss a *derived* - * expression over a key (e.g. `upper(lower(a))` when the key is `lower(a)`, which has - * `sortGroupRef == 0` — it does not match the key textually, only structurally, which - * [isSafeFromGroupingSetNullExtension] is what actually catches). The second is not subsumed by - * [isSafeFromGroupingSetNullExtension] either — that method proves an expression's *result* - * cannot be forced null by null-extending some deeper subexpression, which says nothing about - * the expression *as a whole* being wholesale swapped for `NULL` because it happens to - * structurally repeat the grouping key. The third would miss a bare-`Const` grouping key (e.g. - * `GROUP BY ROLLUP('ALL'::text)`) — a `Const` is [isSafeFromGroupingSetNullExtension] by - * definition (see that method), yet PostgreSQL still null-extends it when it is the grouping - * key, which only the first condition (or, for an unref'd duplicate Const, the second) catches. + * `ressortgroupref` to. Not subsumed by the third condition, which proves only that a result + * cannot be forced null by a *deeper* subexpression being null-extended, not that the whole + * expression is swapped for `NULL` because it structurally repeats the grouping key. + * - [entry]'s expression is not proven [isSafeFromGroupingSetNullExtension]. Alone this misses a + * bare-`Const` grouping key (e.g. `GROUP BY ROLLUP('ALL'::text)`): a `Const` is always + * [isSafeFromGroupingSetNullExtension], yet PostgreSQL still null-extends it when it is itself + * the grouping key — caught by the first or second condition instead. * * @param groupingKeyExpressions the expressions of every entry whose own [TargetEntry.sortGroupRef] * is a grouping key (per [groupingSortGroupRefs]), excluding any that are a bare * [PgNodeExpression.Const] or that [foldsToConst] — PostgreSQL's structural matching - * (`search_indexed_tlist_for_non_var` in `setrefs.c`) explicitly refuses to match a `Const` - * node (see [isSafeFromGroupingSetNullExtension]'s KDoc), and an expression that folds to a - * `Const` before that matching pass runs (e.g. `upper('a')`) is, by the time the pass runs, - * already a `Const` too — true on every supported version (16, 17, and 18): `SELECT - * upper('a') AS u1, upper('a') AS u2, ... GROUP BY ROLLUP(upper('a'))` leaves the un-ref'd - * duplicate `u2` as `'A'`, never `NULL`, in the ROLLUP summary row, unlike a duplicate that - * does not fold (see the `date_trunc` case in [isSafeFromGroupingSetNullExtension]'s KDoc, - * where both occurrences are null-extended). Without this exclusion, a duplicate literal like - * `SELECT 'ALL'::text AS l1, 'ALL'::text AS l2, ... GROUP BY ROLLUP('ALL'::text)` would be - * wrongly forced nullable for `l2` on every supported version: `l2` stays - * `'ALL'`, never `NULL`, even though `l1` (the ref'd occurrence) does become `NULL`. - * - * On PostgreSQL 18, [entry] arrives here already having been run through - * [substituteGroupRteVars] (see [extractColumnNullability]'s own KDoc) — every target-list - * `Var` that PostgreSQL 18's parse-analysis phase rewrote into a reference to the synthesized - * `*GROUP*` RTE has already been resolved back to the real expression it stands for, restoring - * the same tree shape PostgreSQL 16/17 produce directly. `u2`/`l2` therefore arrive here as the - * genuine `FuncExpr`/`Const` PostgreSQL 16/17 always showed, not a bare `Var`, so this exclusion - * rescues them identically on every supported version — see `QueryAnalysisTest`'s `duplicate - * bare Const grouping key stays non-null...` and `duplicate IMMUTABLE-folding call stays - * non-null...` tests, which pin this as an unconditional (not version-branched) assertion. + * (`search_indexed_tlist_for_non_var` in `setrefs.c`) refuses to match a `Const` node, and an + * expression that folds to a `Const` before that matching pass runs (e.g. `upper('a')`) is, by + * then, already a `Const` too: on PostgreSQL 16, 17, and 18, `SELECT upper('a') AS u1, + * upper('a') AS u2, ... GROUP BY ROLLUP(upper('a'))` leaves the un-ref'd duplicate `u2` as + * `'A'`, never `NULL`, unlike a duplicate that does not fold, e.g. + * `date_trunc('month', current_date)`, where both occurrences are null-extended. Without this + * exclusion, a duplicate literal like `SELECT 'ALL'::text AS l1, 'ALL'::text AS l2, ... GROUP BY + * ROLLUP('ALL'::text)` would be wrongly forced nullable for `l2` on every supported version: + * `l2` stays `'ALL'`, never `NULL`, even though `l1` (the ref'd occurrence) does become `NULL`. + * + * On PostgreSQL 18, [entry] arrives here already run through [substituteGroupRteVars], so + * `u2`/`l2` arrive as the genuine `FuncExpr`/`Const` PostgreSQL 16/17 always showed, not a bare + * `Var` referencing the synthesized `*GROUP*` RTE, so this exclusion rescues them identically + * on every supported version. */ private fun isEffectivelyNonNull( entry: TargetEntry, @@ -229,135 +185,58 @@ internal class NodeTreeNullabilityAnalyzer( /** * Returns `true` if [expression] is provably immune to PostgreSQL's GROUPING SETS/CUBE/ROLLUP * null-extension mechanism — i.e. it cannot be the *value* PostgreSQL replaces with `NULL` for a - * row belonging to a grouping set that omits it. - * - * Only meaningful when the enclosing query block has GROUPING SETS, CUBE, or ROLLUP - * ([hasGroupingSets]); see [isEffectivelyNonNull] for how the two nullability conditions combine. - * - * Null-extension is a **structural, planner-level substitution**: PostgreSQL scans the target - * list for stable, non-folded subexpressions that match a grouping key and replaces their - * computed value with `NULL` outright for rows outside that key's grouping set — it does not - * evaluate the subexpression's own semantics first. This means even a construct that is - * *semantically* always non-null under ordinary evaluation (e.g. `EXISTS(...)`, `ARRAY[...]`, - * `IS NULL`) can still be replaced with a literal `NULL` if it matches a grouping key. Whether an - * expression CAN match is governed by two special cases PostgreSQL's matching applies before - * falling through to ordinary structural equality: - * - `Aggref`/`GroupingFunc` — aggregates are illegal inside `GROUP BY`, so no expression - * containing one can itself be a grouping key, and neither can any expression built on top of - * one, because the aggregate/grouping value it depends on cannot be null-extended out from - * under it. - * - `Const` — PostgreSQL's grouping-key matching specifically refuses to match a bare constant - * (there would be no point: a constant is trivially recomputable), so a lone `Const` is never - * itself null-extended. This does not extend to a `Const` wrapped in a non-folded coercion - * chain (e.g. a `text`-to-`timestamptz` cast, which is a real function call, not a no-op) — - * that wrapping expression is a stable, matchable subexpression like any other, and the `Const` - * underneath it does not make it safe. - * - * So `safe(e)` is: [foldsToConst] → safe (a third, independent leg — see that method); a - * NON-`VARIADIC` [PgNodeExpression.FuncExpr] whose function is [isAlwaysNonNull] → safe (a - * fourth, independent leg — see the note near the bottom of this KDoc); otherwise - * `Aggref`/`GroupingFunc` → safe (matches the first special case); `Const` → safe (matches the - * second, though [foldsToConst] already subsumes it); `WindowFunc` → safe iff every child is - * itself safe (a window function can never itself be a grouping key — window functions, like - * aggregates, are illegal inside `GROUP BY` — but unlike `Aggref` it does not get blanket safety: - * its arguments are evaluated over already-grouped, potentially null-extended rows, e.g. - * `first_value(b) OVER (...)` is genuinely nullable — see the ground truth in - * `QueryAnalysisTest`); everything else, **including a bare `Var`**, → safe iff `e`'s parsed - * descendants include at least one `Aggref`/`GroupingFunc`/`WindowFunc` (per the first special - * case — a `WindowFunc` counts here too, since it likewise can never itself be a grouping key) AND - * every parsed child of `e` is itself safe (the whole subtree is dominated by that - * aggregate/window, modulo constants and foldable subexpressions). A bare `Var` has no - * descendants, so it is never safe under this rule (correct: a bare column reference is exactly - * what a grouping key most commonly is, or is derived from). `count(*) + 1` is safe (its `OpExpr` - * has an `Aggref` descendant and both children — `Aggref`, `Const` — are themselves safe); - * `count(*) || some_stable_cast(a_const)` is not safe, because the cast side has no - * `Aggref`/`WindowFunc` descendant and does not [foldsToConst] (a stable cast survives constant - * folding) even though the `||` as a whole has an `Aggref` — safety is required of every child - * independently, not just the subtree as a whole, otherwise a matchable non-aggregate side would - * be missed. + * row belonging to a grouping set that omits it. Only meaningful when [hasGroupingSets] is `true`. + * + * Null-extension is a structural, planner-level substitution: PostgreSQL scans the target list + * for stable, non-folded subexpressions that match a grouping key and replaces their computed + * value with `NULL` outright, without evaluating the subexpression's own semantics — so even a + * construct that is *semantically* always non-null (`EXISTS(...)`, `ARRAY[...]`, `IS NULL`) can + * still be replaced with `NULL` if it structurally matches a grouping key. + * + * Two node kinds are exempt from ever matching a grouping key: `Aggref`/`GroupingFunc` + * (aggregates are illegal inside `GROUP BY`, so nothing built on one can itself be a grouping + * key), and a bare `Const` (PostgreSQL's matching specifically refuses to match a constant) — + * though a `Const` wrapped in a non-folded coercion (e.g. a `text`-to-`timestamptz` cast) is a + * real function call and does not inherit that exemption. + * + * Beyond those two, an expression is safe if it [foldsToConst]; or is a non-`VARIADIC` + * [PgNodeExpression.FuncExpr] whose function [isAlwaysNonNull] (its own result cannot be forced + * `null` by null-extending one of its arguments — e.g. `concat(a, '-')` stays `'-'`, never + * `null`, when `a` alone, not the whole call, is the grouping key, PostgreSQL 16-18; this does not + * apply to a `VARIADIC` call, since `concat(VARIADIC arr)` is `null` when `arr` itself is `null`, + * also PostgreSQL 16-18); or a `WindowFunc` whose every child is itself safe (a window function + * can never itself be a grouping key, but its arguments run over already-grouped, potentially + * null-extended rows — `first_value(b) OVER (...)` is genuinely nullable); or, for everything else + * including a bare `Var`, iff the expression's parsed descendants include at least one + * `Aggref`/`GroupingFunc`/`WindowFunc` and every parsed child is itself safe. A bare `Var` has no + * descendants, so it is never safe under this last rule. [isNonNullIffFirstArgumentNonNull] gets + * its own conditional leg — safe iff its first argument is itself safe, since a `concat_ws` + * separator can independently be a grouping key. [immuneByNoGroupingKeyMatch] is a structurally + * different leg, for constructs with no per-node-kind rule at all, e.g. `now()`. * * This walk is only sound for a [PgNodeExpression] subtype whose parsed representation retains - * every child expression the underlying Postgres node actually has — for a subtype that drops a - * child, this method cannot rule out an unseen child changing the answer. - * [PgNodeExpression.CaseExpr] is the motivating example that is handled faithfully: - * [PgNodeExpression.CaseExpr.testExpression] and [PgNodeExpression.CaseExpr.whenConditions] exist - * on that type purely so this walk (not [isNonNull], which correctly ignores them, since a `CASE` - * result's nullability never depends on its own test/condition expressions) can see a `Var` that - * appears only in a `CASE`'s test expression or a `WHEN` condition, e.g. - * `CASE a WHEN 'x' THEN 1 ELSE 2 END` or `CASE WHEN a = 'x' THEN 1 ELSE 2 END`. - * - * [PgNodeExpression.JsonExpr] and [PgNodeExpression.Unknown] are the two subtypes that drop - * information this method cannot recover by walking harder — [PgNodeExpression.JsonExpr] does not - * retain a `JSON_VALUE`/`JSON_QUERY`/`JSON_EXISTS` `PASSING` clause's values, so a `Var` living - * only there (e.g. `JSON_EXISTS(doc, '\$.a ? (@ == \$v)' PASSING a AS v)`) is invisible; parsing - * the `PASSING` clause was deliberately not attempted (parser work against an unconfirmed node - * shape for marginal precision gain). Both are therefore hardcoded unsafe unconditionally, - * regardless of an `Aggref` elsewhere in the tree — an `Aggref` sibling cannot rescue a subtree - * that might independently contain a hidden, matchable `Var`. The same treatment applies once - * [depth] is exhausted. - * - * A fourth, independent leg alongside [foldsToConst]: a non-`VARIADIC` [PgNodeExpression.FuncExpr] - * whose function is [isAlwaysNonNull] (e.g. `concat` — see - * [PgCatalogLoader.alwaysNonNullFunctionOids]) is safe from having its own result forced `null` - * by a deeper subexpression being null-extended — by that list's own definition, `concat` renders - * a `null` argument as an empty string, so null-extending one of its arguments (e.g. `a` inside - * `concat(a, '-')` when `a` alone, not the whole `concat` call, is the grouping key — PostgreSQL - * 16-18: `concat(a, '-')` stays `'-'`, never `null`, in that case) cannot make - * the call's result `null`. This is a different scenario from `concat(a, '-')` itself being - * null-extended wholesale because it structurally repeats the grouping key expression (e.g. - * `GROUP BY ROLLUP(concat(a, '-'))` null-extends a duplicate, un-ref'd - * occurrence too, not just the one PostgreSQL attached `ressortgroupref` to) — there, - * PostgreSQL's substitution replaces the entire call's result before `concat` ever runs, so its - * argument-null-tolerance is irrelevant and provides no protection. This leg does not (and, from - * inside a single expression's own subtree, structurally cannot) distinguish the two; ruling out - * the second is [isEffectivelyNonNull]'s job via its `groupingKeyExpressions` structural-duplicate - * check, which this leg's safety claim depends on to stay sound. Deferring to ordinary [isNonNull] - * evaluation for the first scenario independently reaches the same conclusion via the identical - * [isAlwaysNonNull] check in its own [PgNodeExpression.FuncExpr] branch. `concat_ws` is - * deliberately not on [isAlwaysNonNull]'s list — despite also being non-strict, it is non-null - * only when its first argument (the separator) is non-null, so it gets no dedicated leg here and - * falls through to the generic aggregate/window domination rule below like any other `FuncExpr`, - * where a `Var` in any of its argument positions — including the separator — correctly makes it - * unsafe; see [PgCatalogLoader.alwaysNonNullFunctionOids]'s KDoc for why this distinction matters. - * The `VARIADIC` exclusion matters for the same reason [isNonNull] excludes it: - * `concat(VARIADIC arr)` is `null` when `arr` itself is `null` (PostgreSQL 16-18) - * — a `VARIADIC` call gets no short-circuit here at all and falls through to the - * generic rule, where its sole argument (the array — a `Var` for a column, or an `ArrayExpr` - * for a literal) is evaluated on its own merits, correctly unsafe if it can be null-extended. - * This does not extend to a function merely on the (much - * larger, strict-only) [isNeverNullForNonNullInput] safe-list — that list only proves totality - * for non-null arguments, and says nothing about whether the function's result stays non-null - * when one of its own arguments is individually null-extended (the first scenario above), which - * is exactly the scenario this leg exists to guard against for the (much narrower) functions - * that are on [isAlwaysNonNull]'s list. - * - * Several `when` branches below answer the same argument-independence question the fourth leg does: - * null-extension substitutes `NULL` for an ARGUMENT's value, so any rule that proves a result - * non-null without consulting its arguments' nullability already answers it. The - * [isNonNullIffFirstArgumentNonNull] branch is the one conditional case — safe iff its first - * argument is itself safe, since a `concat_ws` separator can be a grouping key. - * [immuneByNoGroupingKeyMatch] is a structurally different leg, for constructs with no - * per-node-kind rule at all, e.g. `now()`. - * - * `XML_IS_XMLFOREST` and `XML_IS_XMLPI` are excluded because neither is total over `null` input - * (measurements in [evaluateXmlExpr]); admitting them produced a real wrong non-null — `SELECT - * xmlforest(lower(a) AS q), count(*) FROM t2 GROUP BY ROLLUP(a)`, which PostgreSQL 16, 17 and - * 18 all return `NULL` for in the rollup summary row. PostgreSQL has no equality operator for `xml` - * or for `json`, so neither an `XmlExpr` nor a default-`RETURNING` - * `JSON_OBJECT`/`JSON_ARRAY` (which yields `json`) can itself be a grouping key; `jsonb` does have - * one, so a `RETURNING jsonb` call can be. - * - * The self-match guard runs before every leg because the legs prove only that an expression's own - * result survives null-extension of a deeper subexpression, never that the expression itself is not - * replaced by `NULL` wholesale. Running it at every node the walk reaches, not only at - * [isEffectivelyNonNull]'s entry root, fixes a measured wrong non-null: `SELECT count(*)::text || - * concat(a, b) FROM t2 GROUP BY ROLLUP(concat(a, b))` reported non-null where live PostgreSQL 16 - * and 18 return `NULL`. - * - * @param groupingKeyExpressions the same set [isEffectivelyNonNull] receives; see that method's - * identically-named parameter. + * every child the underlying Postgres node actually has. [PgNodeExpression.JsonExpr] and + * [PgNodeExpression.Unknown] drop information this method cannot recover and are hardcoded unsafe + * unconditionally — a `JSON_EXISTS`'s `PASSING` clause (e.g. `JSON_EXISTS(doc, '\$.a ? (@ == \$v)' + * PASSING a AS v)`) is not parsed, so a `Var` living only there is invisible. The same treatment + * applies once [depth] is exhausted. [PgNodeExpression.CaseExpr.testExpression] and + * [PgNodeExpression.CaseExpr.whenConditions] exist purely so this walk can see a `Var` that + * appears only in a `CASE`'s test/condition, e.g. `CASE a WHEN 'x' THEN 1 ELSE 2 END`. + * + * `XML_IS_XMLFOREST` and `XML_IS_XMLPI` are excluded from the always-safe `XmlExpr` case because + * neither is total over `null` input: `SELECT xmlforest(lower(a) AS q), count(*) FROM t2 GROUP BY + * ROLLUP(a)` returns `NULL` in the rollup summary row on PostgreSQL 16, 17, and 18. PostgreSQL has + * no equality operator for `xml` or for default-`RETURNING` `json`, so neither an `XmlExpr` nor a + * `JSON_OBJECT`/`JSON_ARRAY` yielding `json` can itself be a grouping key; `jsonb` does have one. + * + * The self-match guard (`expression in groupingKeyExpressions`) runs at every node the walk + * reaches, not only the entry root, because a match can occur on a nested subexpression: `SELECT + * count(*)::text || concat(a, b) FROM t2 GROUP BY ROLLUP(concat(a, b))` returns `NULL` on live + * PostgreSQL 16 and 18, which only checking the root would miss. + * + * @param groupingKeyExpressions the same set [isEffectivelyNonNull] receives. * @param depth remaining recursion budget, mirroring [MAX_EXPRESSION_DEPTH]; returns `false` - * (assume unsafe — i.e. possibly null-extended) once exhausted + * (assume unsafe) once exhausted */ internal fun isSafeFromGroupingSetNullExtension( expression: PgNodeExpression, @@ -411,18 +290,17 @@ internal class NodeTreeNullabilityAnalyzer( * nothing GROUPING SETS/CUBE/ROLLUP null-extension could act on: no [PgNodeExpression.Var], no * lossily-parsed node, and no structural match against [groupingKeyExpressions]. A `Var`-free * expression's value is fixed for the row whichever grouping set that row belongs to, so `now()` - * under `GROUP BY ROLLUP(a)` is immune without [containsDominatingConstruct]. + * under `GROUP BY ROLLUP(a)` is immune. * - * The lossy nodes are [PgNodeExpression.JsonExpr] and [PgNodeExpression.Unknown], for the reason - * [isSafeFromGroupingSetNullExtension] gives, plus [PgNodeExpression.SubLink], which that method - * need not name: [PgNodeExpression.SubLink.subselectBlock] keeps the subselect as unparsed raw text - * and [safetyWalkChildren] walks only [PgNodeExpression.SubLink.outerOperand], so a correlated - * `Var` inside the subselect is invisible to the `Var` check. + * The lossy nodes are [PgNodeExpression.JsonExpr] and [PgNodeExpression.Unknown], plus + * [PgNodeExpression.SubLink]: [PgNodeExpression.SubLink.subselectBlock] keeps the subselect as + * unparsed raw text and [safetyWalkChildren] walks only [PgNodeExpression.SubLink.outerOperand], + * so a correlated `Var` inside the subselect is invisible to the `Var` check. * - * The match is sound in this direction: [PgNodeExpression]'s subtypes retain a SUBSET of the fields - * PostgreSQL's own `equal()` compares (e.g. [PgNodeExpression.Const] keeps only `isNull`), so this - * class's structural equality is COARSER than PostgreSQL's — a false "no match" verdict can never - * arise from a field this parser dropped. + * The structural-equality match is sound in this direction only: [PgNodeExpression]'s subtypes + * retain a SUBSET of the fields PostgreSQL's own `equal()` compares (e.g. + * [PgNodeExpression.Const] keeps only `isNull`), so this class's equality is COARSER than + * PostgreSQL's — a false "no match" verdict can never arise from a field this parser dropped. * * @param depth remaining recursion budget, mirroring [MAX_EXPRESSION_DEPTH]; returns `false` (not * provably immune) once exhausted. @@ -444,31 +322,26 @@ internal class NodeTreeNullabilityAnalyzer( * Returns `true` if [expression] provably constant-folds by the time PostgreSQL's planner reaches * the grouping-set null-extension substitution — i.e. it is a [PgNodeExpression.Const], or an * IMMUTABLE, non-set-returning function/operator call whose every argument itself [foldsToConst], - * or a [PgNodeExpression.RelabelType] (a no-op type reinterpretation, not a function call) over - * one, or a [PgNodeExpression.ArrayExpr] whose every element does. + * or a [PgNodeExpression.RelabelType] (a no-op type reinterpretation) over one, or a + * [PgNodeExpression.ArrayExpr] whose every element does. * - * This matters because PostgreSQL's grouping-set null-extension substitution - * (`search_indexed_tlist_for_non_var` in `setrefs.c`) runs at the end of planning and explicitly - * refuses to match a `Const` (`if (IsA(node, Const)) return NULL`), while constant folding itself - * (`eval_const_expressions`, from `preprocess_expression`) runs early, well before that - * substitution. An expression that will fold to a `Const` by the time the substitution runs was - * therefore never a candidate for it — safe regardless of whether it contains an - * `Aggref`/`GroupingFunc`/`WindowFunc`, unlike the general rule in [isSafeFromGroupingSetNullExtension]. + * PostgreSQL's null-extension substitution (`search_indexed_tlist_for_non_var` in `setrefs.c`) + * runs at the end of planning and refuses to match a `Const`, while constant folding + * (`eval_const_expressions`) runs early, well before that substitution — so an expression that + * will fold to a `Const` by the time the substitution runs was never a candidate for it. * * IMMUTABLE only. A STABLE function — e.g. `date_trunc('month', current_date)`, which depends on - * the current date — is not constant-folded, survives to become a genuine, matchable - * subexpression, and PostgreSQL does null-extend it when it matches a grouping key. VOLATILE is - * not safe either — `GROUP BY random()` is legal SQL, and the key-matching itself is structural - * (`equal()`), not a volatility check. + * the current date — is not constant-folded and PostgreSQL does null-extend it when it matches a + * grouping key. VOLATILE is not safe either — `GROUP BY random()` is legal SQL, and the + * key-matching itself is structural (`equal()`), not a volatility check. * * [PgNodeExpression.CoerceViaIo], [PgNodeExpression.CoerceToDomain], * [PgNodeExpression.ArrayCoerceExpr], [PgNodeExpression.RowExpr], * [PgNodeExpression.SqlValueFunction], and [PgNodeExpression.NextValExpr] are deliberately not * treated as foldable: none of them expose a function/operator OID this class can check - * immutability for (unlike [PgNodeExpression.FuncExpr]/[PgNodeExpression.OpExpr]), so treating any - * of them as folding would be an unverified guess. This matters concretely for - * [PgNodeExpression.CoerceViaIo]: an I/O-based cast function can itself be STABLE (e.g. - * `timestamptz`'s output function depends on the session's `TimeZone` setting). + * immutability for. This matters concretely for [PgNodeExpression.CoerceViaIo]: an I/O-based cast + * function can itself be STABLE (e.g. `timestamptz`'s output function depends on the session's + * `TimeZone` setting). * * @param depth remaining recursion budget, mirroring [MAX_EXPRESSION_DEPTH]; returns `false` (not * provably folding) once exhausted @@ -507,54 +380,17 @@ internal class NodeTreeNullabilityAnalyzer( } /** - * Returns the immediate parsed child expressions of [expression] for - * [isSafeFromGroupingSetNullExtension] and [containsDominatingConstruct]. - * - * [PgNodeExpression.Var], [PgNodeExpression.Const], [PgNodeExpression.Aggref], - * [PgNodeExpression.GroupingFunc], [PgNodeExpression.SqlValueFunction], - * [PgNodeExpression.NextValExpr], [PgNodeExpression.JsonExpr], and [PgNodeExpression.Unknown] are - * resolved directly by their callers without consulting this method (each is either a genuine - * leaf or hardcoded by the lossy-parsing rule), so they return an empty list here defensively - * rather than being omitted from the `when`. + * Returns the child expressions of [expression] relevant to [isSafeFromGroupingSetNullExtension] + * and [containsDominatingConstruct], deliberately narrower than the full structural + * [PgNodeExpression.children]: + * - [PgNodeExpression.Aggref] and [PgNodeExpression.GroupingFunc] are terminal for the domination + * check — finding one already answers the question, so their own arguments are never walked. + * - [PgNodeExpression.JsonExpr] is lossy-parsed and hardcoded unsafe by its callers, so its + * children are never consulted here either. */ private fun safetyWalkChildren(expression: PgNodeExpression): List = when (expression) { - is PgNodeExpression.FuncExpr -> expression.arguments - is PgNodeExpression.OpExpr -> expression.arguments - is PgNodeExpression.ScalarArrayOpExpr -> expression.arguments - is PgNodeExpression.CoalesceExpr -> expression.arguments - is PgNodeExpression.NullIfExpr -> expression.arguments - is PgNodeExpression.MinMaxExpr -> expression.arguments - is PgNodeExpression.WindowFunc -> expression.arguments - is PgNodeExpression.SubLink -> listOfNotNull(expression.outerOperand) - is PgNodeExpression.CaseExpr -> - expression.resultExpressions + - listOfNotNull(expression.defaultResult, expression.testExpression) + - expression.whenConditions - - is PgNodeExpression.BoolExpr -> expression.arguments - is PgNodeExpression.RelabelType -> listOf(expression.argument) - is PgNodeExpression.CoerceViaIo -> listOf(expression.argument) - is PgNodeExpression.ArrayCoerceExpr -> listOf(expression.argument) - is PgNodeExpression.CollateExpr -> listOf(expression.argument) - is PgNodeExpression.CoerceToDomain -> listOf(expression.argument) - is PgNodeExpression.NullTest -> listOf(expression.argument) - is PgNodeExpression.BooleanTest -> listOf(expression.argument) - is PgNodeExpression.DistinctExpr -> expression.arguments - is PgNodeExpression.ArrayExpr -> expression.elements - is PgNodeExpression.RowExpr -> expression.arguments - is PgNodeExpression.FieldSelect -> listOf(expression.argument) - is PgNodeExpression.JsonIsPredicate -> listOf(expression.argument) - is PgNodeExpression.JsonConstructorExpr -> expression.arguments + listOfNotNull(expression.function) - is PgNodeExpression.XmlExpr -> expression.arguments - is PgNodeExpression.Var, - is PgNodeExpression.Const, - is PgNodeExpression.Aggref, - is PgNodeExpression.GroupingFunc, - is PgNodeExpression.SqlValueFunction, - is PgNodeExpression.NextValExpr, - is PgNodeExpression.JsonExpr, - is PgNodeExpression.Unknown, - -> emptyList() + is PgNodeExpression.Aggref, is PgNodeExpression.GroupingFunc, is PgNodeExpression.JsonExpr -> emptyList() + else -> expression.children } /** @@ -573,23 +409,14 @@ internal class NodeTreeNullabilityAnalyzer( val recurse = { expr: PgNodeExpression -> isNonNull(expr, depth - 1) } return when (expression) { is PgNodeExpression.Var -> - // A PostgreSQL 18+ RETURNING WITH (OLD AS o, ...) reference to the OLD row must never be - // treated as non-null on the strength of the source column's own NOT NULL constraint or - // outer-join structure — see PgNodeExpression.Var.returningType's KDoc for why the OLD row - // itself may not exist for this result row at all (e.g. a MERGE ... WHEN NOT MATCHED THEN - // INSERT action), a fact neither of those signals captures. The same applies to NEW - // whenever forceNewNullable says so — see that constructor parameter's KDoc. + // An OLD row may not exist for this result row at all (e.g. MERGE ... WHEN NOT MATCHED + // THEN INSERT), so its NOT NULL constraint and outer-join structure prove nothing; the + // same applies to NEW whenever forceNewNullable says so. // // levelsUp == 0 is required because a Var with levelsUp > 0 indexes an ENCLOSING query's - // range table (see PgNodeExpression.Var.levelsUp's KDoc), not the current block's — this - // block's isSourceColumnNotNull, groupRteMap, and qual narrowing are all keyed against the - // CURRENT block's varnos, so resolving an outer-level varno against them would be sound in - // neither direction (a collision could read the wrong column's constraint entirely, in - // either the non-null or nullable direction). qualProvenNonNullVars already excludes these - // for its own WHERE-clause narrowing (see that method's KDoc); this makes the target-entry - // evaluation path agree, rather than silently trusting a levelsUp > 0 Var it happens to - // reach through an untouched path (e.g. an ANY_SUBLINK's own subselect target list — see - // isSubLinkSubqueryColumnNotNull). + // range table, not the current block's — this block's isSourceColumnNotNull, groupRteMap, + // and qual narrowing are all keyed against the CURRENT block's varnos, so resolving an + // outer-level varno against them could read the wrong column's constraint entirely. expression.levelsUp == 0 && expression.returningType != PgNodeExpression.VAR_RETURNING_TYPE_OLD && !(expression.returningType == PgNodeExpression.VAR_RETURNING_TYPE_NEW && forceNewNullable) && @@ -600,31 +427,19 @@ internal class NodeTreeNullabilityAnalyzer( is PgNodeExpression.FuncExpr -> if (expression.isVariadic) { if (isAlwaysNonNull(expression.functionOid) || isNonNullIffFirstArgumentNonNull(expression.functionOid)) { - // VARIADIC passes the array argument itself as one value, not exploded into - // elements (see PgNodeExpression.FuncExpr.isVariadic's KDoc) — isAlwaysNonNull's - // "regardless of any argument" and isNonNullIffFirstArgumentNonNull's "only the - // first argument matters" both assume the ordinary calling form and are unsound - // here: concat(VARIADIC arr) and concat_ws(',', VARIADIC arr) are both null when - // arr itself is null (PostgreSQL 16-18). Requiring every - // argument non-null is sound for both functions in this form (the separator is - // still one of the arguments) and preserves real precision (concat_ws(',', - // VARIADIC ARRAY['a', NULL]) is 'a', still non-null). + // VARIADIC passes the array argument itself as one value, not exploded into elements, + // so both guarantees are unsound here as stated: concat(VARIADIC arr) and + // concat_ws(',', VARIADIC arr) are both null when arr itself is null (PostgreSQL + // 16-18). Requiring every argument non-null is sound for both in this form (the + // separator is still one of the arguments) and preserves real precision + // (concat_ws(',', VARIADIC ARRAY['a', NULL]) is 'a', still non-null). expression.arguments.all(recurse) } else { - // Deliberately does not fall through to the isStrict/isNeverNullForNonNullInput leg - // below. That safe-list's "total on non-null input" guarantee (see - // PgCatalogLoader.neverNullForNonNullInputOids's KDoc and SafeListSweepTest) was - // verified for the ordinary, element-wise calling convention. For a VARIADIC call, - // "every argument non-null" only means the array Datum itself is non-null — - // recurse() on the array (an ArrayExpr) is unconditionally true regardless of NULL - // elements inside it (an array container is never NULL merely because one of its - // elements is), so that leg would prove nothing about NULL elements even if - // reached, and no per-function verification exists for whether an internal NULL - // element could still make a safe-listed function return NULL when invoked this - // way. Not reachable today — verified empirically that no name on - // NEVER_NULL_FUNCTION_SIGNATURES resolves to a `provariadic <> 0` function on - // PostgreSQL 16, 17, or 18 — but must not silently start trusting the list the - // moment a variadic name is added to it. + // Does not fall through to isStrict/isNeverNullForNonNullInput: that safe-list's + // "total on non-null input" guarantee was verified only for the ordinary, element-wise + // calling convention. For a VARIADIC call, recurse() on the array argument is + // unconditionally true regardless of NULL elements inside it, so that leg would prove + // nothing about an internal NULL element even if reached. false } } else { @@ -658,33 +473,18 @@ internal class NodeTreeNullabilityAnalyzer( is PgNodeExpression.SubLink -> expression.subLinkType == PgNodeExpression.SUBLINK_TYPE_EXISTS || expression.subLinkType == PgNodeExpression.SUBLINK_TYPE_ARRAY || - // ANY_SUBLINK (`x = ANY (subquery)` / `x IN (subquery)`) is three-valued: PostgreSQL - // returns NULL, not FALSE, when the subquery yields a NULL row and no row matches — - // on PostgreSQL 17: `CREATE TABLE t (id INT PRIMARY KEY, a TEXT NOT NULL); - // CREATE TABLE u (v TEXT); INSERT INTO t VALUES (1,'x'); INSERT INTO u VALUES ('q'), - // (NULL); SELECT a = ANY (SELECT v FROM u) FROM t;` is NULL, not FALSE. Proving a - // non-null result therefore requires all three of: the outer operand is non-null (an - // ANY_SUBLINK with a null outer operand is NULL outright, same as any comparison); the - // comparison operator behind the sublink is both isStrict and isNeverNullForNonNullInput - // — a non-strict or non-total operator could itself manufacture a NULL from non-null - // operands, same two-predicate proof OpExpr/ScalarArrayOpExpr require above; and the - // subquery's single output column is itself provably non-null — a NULL row in the - // subquery is exactly what makes the whole expression NULL when no row matches, per the - // repro above. testExpressionOperatorOid is null for the multi-column `(a, b) IN (SELECT - // p, q FROM w)` row-comparison form (a BOOLEXPR testexpr with no single top-level - // operator), so that form always falls through to nullable here rather than needing its - // own special case. Ordered cheapest-first: subLinkType and outerOperand are already - // computed above; the two OID predicates are cheap map lookups; isSubLinkSubqueryColumnNotNull - // is the only leg that re-enters full query-block analysis, so it is checked last. + // ANY_SUBLINK (`x = ANY (subquery)` / `x IN (subquery)`) is three-valued: on PostgreSQL + // 17, `a = ANY (SELECT v FROM u)` is NULL, not FALSE, when u.v is nullable, u has no + // matching row, and a NULL row is present. Proving non-null requires all three: the + // outer operand is non-null; the comparison operator is both isStrict and + // isNeverNullForNonNullInput; and the subquery's single output column is itself provably + // non-null. testExpressionOperatorOid is null for the multi-column `(a, b) IN (SELECT p, + // q FROM w)` row-comparison form, so that form always falls through to nullable. // // ALL_SUBLINK (`x op ALL (subquery)`) gets the identical proof, being ANY's dual over AND - // instead of OR: `x op ALL (S)` is NULL only when some comparison is NULL and none is FALSE, - // which the same three conditions rule out. An empty S is TRUE for ALL (FALSE for ANY) — - // non-null either way, so no empty-subquery case needs handling. `NOT IN` desugars to a - // BOOLEXPR not around an ANY_SUBLINK, never an ALL_SUBLINK, so a negation cannot hide inside - // an ALL testexpr. The multi-column row-comparison ALL forms fail automatically, because - // testExpressionOperatorOid is null for both their shapes — see - // PgNodeExpression.SubLink.testExpressionOperatorOid. + // instead of OR. An empty subquery is TRUE for ALL (FALSE for ANY) — non-null either way. + // `NOT IN` desugars to a BOOLEXPR not around an ANY_SUBLINK, never an ALL_SUBLINK. The + // multi-column row-comparison ALL forms fail automatically for the same reason as above. // // ROWCOMPARE_SUBLINK (`(a, id) < (SELECT v, 1 FROM u)`, no ALL/ANY keyword) is excluded // despite sharing SUBLINK's shape: an EMPTY subquery yields NULL for ROWCOMPARE, so no @@ -764,8 +564,7 @@ internal class NodeTreeNullabilityAnalyzer( * [PgNodeExpression.JsonConstructorExpr.function] is recursed into instead, reaching the existing * rules that already report an aggregate over an empty group nullable. * - `PARSE`/`SCALAR`/`SERIALIZE`: strict single-argument constructs, non-null only when there is an - * argument and every argument is non-null. This is the original bug report: `JSON_SERIALIZE` was - * reported non-null regardless of its argument. + * argument and every argument is non-null. * - Any other code falls through to `false` (nullable), the safe default. */ private fun evaluateJsonConstructorExpr( @@ -817,31 +616,21 @@ internal class NodeTreeNullabilityAnalyzer( * 18) to make a `JSON_QUERY` `ON EMPTY`/`ON ERROR` clause produce a definite, non-null outcome — * an allow-list. An unrecognized code defaults to nullable, the safe direction. * - * The four allowed codes and why each is safe: - * - [PgNodeExpression.JSON_BEHAVIOR_ERROR]: raises a runtime error rather than returning a value at - * all for the row. `:expr` is absent for this code, so - * [PgNodeExpression.JsonExpr.onEmptyDefault]/[PgNodeExpression.JsonExpr.onErrorDefault] parses to - * `null` here, and `null?.let(recurse) != false` is `true` unconditionally — no default - * expression exists to recurse into. + * The four allowed codes: + * - [PgNodeExpression.JSON_BEHAVIOR_ERROR]: raises a runtime error rather than returning a value. * - [PgNodeExpression.JSON_BEHAVIOR_EMPTY_ARRAY]/[PgNodeExpression.JSON_BEHAVIOR_EMPTY_OBJECT]: * substitute Postgres's own internal `[]`/`{}` `jsonb` constant, never a user-supplied - * expression. `:expr` for these codes is always a `{CONST ... :constisnull - * false ...}` block, so `recurse` on it is unconditionally `true` — checked anyway, defensively, - * rather than special-cased to skip [PgNodeExpression.JsonExpr.onEmptyDefault]/`onErrorDefault` - * entirely. - * - [PgNodeExpression.JSON_BEHAVIOR_DEFAULT]: the one code among these four backed by a genuinely - * user-supplied expression (`DEFAULT expr ON EMPTY`/`ON ERROR`) — it always carries - * a real `:expr` block, which is why [emptyOk]/[errorOk] recurse into it rather than trusting the - * behavior code alone; `DEFAULT null::jsonb ON EMPTY` is legal and must not be treated as - * non-null. + * expression. + * - [PgNodeExpression.JSON_BEHAVIOR_DEFAULT]: the one code backed by a genuinely user-supplied + * expression (`DEFAULT expr ON EMPTY`/`ON ERROR`), which is why [emptyOk]/[errorOk] recurse into + * it rather than trusting the behavior code alone; `DEFAULT null::jsonb ON EMPTY` is legal and + * must not be treated as non-null. * * Deliberately not on this list: [PgNodeExpression.JSON_BEHAVIOR_NULL] (explicitly nullable by - * definition); `JSON_BEHAVIOR_TRUE`/`FALSE`/`UNKNOWN` — Postgres rejects all - * three for a `JSON_QUERY` `ON EMPTY`/`ON ERROR` clause outright, so they never appear here; and - * `JSON_TABLE`'s per-column `ON EMPTY`/`ON ERROR` — a `JSON_TABLE` column - * resolves to a plain `VAR` against an `RTE_TABLEFUNC` range-table entry in the outer query's - * target list, never a [PgNodeExpression.JsonExpr] node this method ever sees, so this allow-list - * has no bearing on it either way. + * definition); `JSON_BEHAVIOR_TRUE`/`FALSE`/`UNKNOWN` (Postgres rejects all three for a + * `JSON_QUERY` `ON EMPTY`/`ON ERROR` clause, so they never appear here); and `JSON_TABLE`'s + * per-column `ON EMPTY`/`ON ERROR` (a `JSON_TABLE` column resolves to a plain `VAR`, never a + * [PgNodeExpression.JsonExpr] this method ever sees). */ private fun isKnownNonNullJsonBehavior(behaviorType: Int): Boolean = behaviorType == PgNodeExpression.JSON_BEHAVIOR_ERROR || @@ -854,14 +643,12 @@ internal class NodeTreeNullabilityAnalyzer( * make `JSON_EXISTS`'s `ON ERROR` clause produce a definite, non-null (`true`/`false`) outcome, or * raise an error rather than returning a value at all. `JSON_EXISTS` has no `ON EMPTY` clause. * - * [PgNodeExpression.JSON_BEHAVIOR_TRUE]/[PgNodeExpression.JSON_BEHAVIOR_FALSE] are the codes for an - * explicit `TRUE`/`FALSE ON ERROR` clause. With no `ON ERROR` clause written at - * all, Postgres materializes `:btype 4` ([PgNodeExpression.JSON_BEHAVIOR_FALSE]) — the SQL-standard - * default — so an absent clause is exactly as safe as writing `FALSE ON ERROR` explicitly. - * Deliberately not on this list: [PgNodeExpression.JSON_BEHAVIOR_UNKNOWN], which produces a - * genuine SQL NULL on a path error, and [PgNodeExpression.JSON_BEHAVIOR_NULL]/ - * `EMPTY_ARRAY`/`EMPTY_OBJECT`/`DEFAULT`, which Postgres's parser rejects outright for - * `JSON_EXISTS`'s `ON ERROR` clause and so never appear here. + * With no `ON ERROR` clause written at all, Postgres materializes + * [PgNodeExpression.JSON_BEHAVIOR_FALSE] — the SQL-standard default — so an absent clause is + * exactly as safe as writing `FALSE ON ERROR` explicitly. Deliberately not on this list: + * [PgNodeExpression.JSON_BEHAVIOR_UNKNOWN], which produces a genuine SQL NULL on a path error, and + * [PgNodeExpression.JSON_BEHAVIOR_NULL]/`EMPTY_ARRAY`/`EMPTY_OBJECT`/`DEFAULT`, which Postgres's + * parser rejects outright for `JSON_EXISTS`'s `ON ERROR` clause. */ private fun isKnownNonNullJsonExistsErrorBehavior(behaviorType: Int): Boolean = behaviorType == PgNodeExpression.JSON_BEHAVIOR_ERROR || @@ -875,17 +662,12 @@ internal class NodeTreeNullabilityAnalyzer( * - `xmlelement(name e, NULL::text)` is not `null` — a null child renders as empty content, and a * null `xmlattributes` value omits that attribute, so the element tag itself always materializes. * - `xmlforest(NULL::text AS q)` is `null`, while `xmlforest(NULL::text AS q, 'x' AS r)` is not — - * a null field is omitted and the result nulls only once every field is gone, hence - * [Iterable.any]. `xmlforest()` is a syntax error, and `any` on an empty list would answer - * `false` (nullable) anyway. - * - `xmlpi(name php, NULL::text)` is `null`, while the content-less `xmlpi(name php)` is not, which - * [Iterable.all] states exactly: vacuously `true` for the zero-argument form, content required - * otherwise. + * a null field is omitted and the result nulls only once every field is gone. + * - `xmlpi(name php, NULL::text)` is `null`, while the content-less `xmlpi(name php)` is not. * - * [PgNodeExpression.XmlExpr.arguments] merges the node's `:named_args` with its `:args` (see - * `PgNodeTreeParser.parseXmlExpr`), so an `XMLFOREST` field value — which lives in `:named_args` — - * is visible to the [Iterable.any] check rather than silently absent. Any other op code falls - * through to `false` (nullable), the safe default. + * [PgNodeExpression.XmlExpr.arguments] merges the node's `:named_args` with its `:args`, so an + * `XMLFOREST` field value — which lives in `:named_args` — is visible to the check rather than + * silently absent. Any other op code falls through to `false` (nullable), the safe default. */ private fun evaluateXmlExpr(expression: PgNodeExpression.XmlExpr, recurse: (PgNodeExpression) -> Boolean): Boolean = when (expression.op) { @@ -934,29 +716,14 @@ internal class NodeTreeNullabilityAnalyzer( /** * `true` if [expression] contains an ordinary `Var` (`returningType == 0` — i.e. not an `OLD` - * or `NEW` reference, which carry their own independent, already-safe handling — see - * [PgNodeExpression.Var.returningType]'s KDoc) whose `varno` is anything other than - * [relationVarno]. - * - * Used by [PgCatalogLoader.mergeAbsentVarnos]'s caller to decide whether a `MERGE`'s - * `RETURNING` list needs per-relation match-optionality resolved AT ALL: a `RETURNING` that - * only reads the target relation's own columns (always present, whichever `WHEN` clause - * matched) or `OLD`/`NEW` references (already forced nullable/handled independently by - * [PgNodeExpression.Var.returningType]) never needs [PgCatalogLoader.mergeAbsentVarnos]'s - * `EXPLAIN` resolution at all — which matters because that resolution can itself fail to - * attribute a `MERGE`'s join (e.g. a non-table `USING` source, such as a `VALUES` list) even - * when the `RETURNING` list never actually depended on knowing which side that join favors. + * or `NEW` reference, which carry their own independent, already-safe handling) whose `varno` + * is anything other than [relationVarno]. * - * Exhausts every [PgNodeExpression] variant explicitly, so the compiler's own exhaustiveness - * check over the sealed [PgNodeExpression] hierarchy guarantees every node type is listed here. - * That guarantee is necessary but not sufficient: exhaustiveness only proves no variant was - * left out of the `when`, not that a listed variant's own child expressions are walked. A - * variant that genuinely carries no [PgNodeExpression] child (e.g. [PgNodeExpression.Const]) - * belongs in the childless branch; one that does (e.g. [PgNodeExpression.JsonExpr]'s - * `argument`) must recurse into every such child, or a `Var` buried inside it silently - * disappears from this check — this exact mistake, for [PgNodeExpression.JsonExpr], is what - * let a `MERGE`'s `RETURNING JSON_QUERY(source.column, ...)` skip `EXPLAIN` resolution and - * report a genuinely nullable expression as not null. + * A `RETURNING` that only reads the target relation's own columns (always present, whichever + * `WHEN` clause matched) or `OLD`/`NEW` references never needs a `MERGE`'s per-relation + * match-optionality resolved via `EXPLAIN` at all. Every non-`Var` branch delegates to + * [PgNodeExpression.children], so no variant can silently keep a child unwalked — e.g. a + * `RETURNING JSON_QUERY(source.column, ...)` is still walked into. * * @param depth remaining recursion budget; exhausting it answers `true` (needs resolving) * rather than `false`, the same fail-toward-conservative default every depth guard in this @@ -971,55 +738,9 @@ internal class NodeTreeNullabilityAnalyzer( val recurse = { childExpression: PgNodeExpression -> containsVarOutsideRelation(childExpression, relationVarno, depth - 1) } - return when (expression) { - is PgNodeExpression.Var -> - expression.returningType == PgNodeExpression.VAR_RETURNING_TYPE_NORMAL && - expression.varno != relationVarno - - is PgNodeExpression.Const, - is PgNodeExpression.SqlValueFunction, - is PgNodeExpression.NextValExpr, - is PgNodeExpression.Unknown, - -> false - - is PgNodeExpression.FuncExpr -> expression.arguments.any(recurse) - is PgNodeExpression.OpExpr -> expression.arguments.any(recurse) - is PgNodeExpression.ScalarArrayOpExpr -> expression.arguments.any(recurse) - is PgNodeExpression.CoalesceExpr -> expression.arguments.any(recurse) - is PgNodeExpression.NullIfExpr -> expression.arguments.any(recurse) - is PgNodeExpression.MinMaxExpr -> expression.arguments.any(recurse) - is PgNodeExpression.WindowFunc -> expression.arguments.any(recurse) - is PgNodeExpression.Aggref -> expression.arguments.any(recurse) - is PgNodeExpression.GroupingFunc -> expression.arguments.any(recurse) - is PgNodeExpression.JsonExpr -> - recurse(expression.argument) || - expression.onEmptyDefault?.let(recurse) == true || - expression.onErrorDefault?.let(recurse) == true - - is PgNodeExpression.SubLink -> expression.outerOperand?.let(recurse) == true - is PgNodeExpression.CaseExpr -> - expression.resultExpressions.any(recurse) || - listOfNotNull(expression.defaultResult, expression.testExpression).any(recurse) || - expression.whenConditions.any(recurse) - - is PgNodeExpression.BoolExpr -> expression.arguments.any(recurse) - is PgNodeExpression.RelabelType -> recurse(expression.argument) - is PgNodeExpression.CoerceViaIo -> recurse(expression.argument) - is PgNodeExpression.ArrayCoerceExpr -> recurse(expression.argument) - is PgNodeExpression.CollateExpr -> recurse(expression.argument) - is PgNodeExpression.CoerceToDomain -> recurse(expression.argument) - is PgNodeExpression.NullTest -> recurse(expression.argument) - is PgNodeExpression.BooleanTest -> recurse(expression.argument) - is PgNodeExpression.DistinctExpr -> expression.arguments.any(recurse) - is PgNodeExpression.ArrayExpr -> expression.elements.any(recurse) - is PgNodeExpression.RowExpr -> expression.arguments.any(recurse) - is PgNodeExpression.FieldSelect -> recurse(expression.argument) - is PgNodeExpression.JsonIsPredicate -> recurse(expression.argument) - is PgNodeExpression.JsonConstructorExpr -> - expression.arguments.any(recurse) || expression.function?.let(recurse) == true - - is PgNodeExpression.XmlExpr -> expression.arguments.any(recurse) - } + if (expression !is PgNodeExpression.Var) return expression.children.any(recurse) + return expression.returningType == PgNodeExpression.VAR_RETURNING_TYPE_NORMAL && + expression.varno != relationVarno } /** diff --git a/generator/src/main/kotlin/norm/generator/NodeTreeProvenance.kt b/generator/src/main/kotlin/norm/generator/NodeTreeProvenance.kt index ccd0f19e..8106bab8 100644 --- a/generator/src/main/kotlin/norm/generator/NodeTreeProvenance.kt +++ b/generator/src/main/kotlin/norm/generator/NodeTreeProvenance.kt @@ -5,8 +5,7 @@ package norm.generator * * Not required to prevent a true infinite loop — PostgreSQL's grammar already forbids a CTE * referencing a later-declared one, so no reference cycle exists among the CTEs this resolver - * enters. This is a defensive bound on stack/work for a pathologically long chain of CTEs, the same - * role [ColumnNullabilityAnalyzer]'s `VIEW_NULLABILITY_RECURSION_DEPTH_BUDGET` plays for view chains. + * enters. This is a defensive bound on stack/work for a pathologically long chain of CTEs. */ private const val MAX_PROVENANCE_CHAIN_DEPTH = 50 @@ -68,9 +67,8 @@ internal class NodeTreeProvenanceResolver(private val parser: PgNodeTreeParser = * Resolves every non-junk output column of [nodeTreeText], in `SELECT`/`RETURNING` order, to * either its [NodeTreeColumnProvenance] or `null` (no CTE provenance for that column). * - * Reads `:returningList` first, falling back to `:targetList` — the same order - * [ColumnNullabilityAnalyzer.analyzeNodeTree] uses, since a topmost `RETURNING` populates both and - * `:targetList` there holds the values being written, not the columns being returned. + * Reads `:returningList` first, falling back to `:targetList`: a topmost `RETURNING` populates + * both, and `:targetList` there holds the values being written, not the columns being returned. * * @return one entry per non-junk output column, in position order; every entry is `null` when * [nodeTreeText]'s outermost statement has a top-level set operation. @@ -80,7 +78,7 @@ internal class NodeTreeProvenanceResolver(private val parser: PgNodeTreeParser = if (entries.isEmpty()) return emptyList() if (parser.hasSetOperations(nodeTreeText)) return entries.map { null } // A single-element stack: nodeTreeText's own :cteList is the only scope until the walk - // descends into a CTE body — see resolveVar for why the stack grows from there. + // descends into a CTE body. val outermostScope = listOf(parser.parseCteList(nodeTreeText).associateBy { it.name }) return entries.map { entry -> resolveVar(entry.expression, nodeTreeText, outermostScope) } } @@ -107,13 +105,11 @@ internal class NodeTreeProvenanceResolver(private val parser: PgNodeTreeParser = * * [scopeStack] tracks lexical `WITH`-clause nesting: index `0` is [queryBlock]'s own `:cteList`, * index `1` the block one level up that declared it, and so on. Scope belongs to a CTE's - * declaration site, not to the hop path taken to reach it — hopping into a sibling CTE declared in - * the same `:cteList` is not a nesting level, matching PostgreSQL's own `:ctelevelsup` for that - * reference. So resolving a reference against `scopeStack[reference.ctelevelsup]` and then - * entering that CTE's body rebuilds the stack as `scopeStack.drop(reference.ctelevelsup)` with the - * body's own `:cteList` pushed on front, rather than prepending onto the full accumulated - * [scopeStack]; otherwise stale frames attribute a chained reference to the wrong same-named CTE, - * and a chain of three or more sibling CTEs resolves to nothing. + * declaration site, not to the hop path taken to reach it, so entering a CTE's body rebuilds the + * stack as `scopeStack.drop(reference.ctelevelsup)` with the body's own `:cteList` pushed on + * front, rather than prepending onto the full accumulated [scopeStack]; prepending onto the full + * stack instead would attribute a chained reference to the wrong same-named CTE for a chain of + * three or more sibling CTEs. * * A `:ctelevelsup` deeper than [scopeStack] bails rather than reading past what has been tracked — * a real reference's levelsup can never exceed the number of `WITH` clauses actually enclosing it. @@ -157,8 +153,6 @@ internal class NodeTreeProvenanceResolver(private val parser: PgNodeTreeParser = } currentQueryBlock = definition.queryBlock val ownScope = parser.parseCteList(definition.queryBlock).associateBy { it.name } - // drop, not prepend-onto-the-full-stack: see this method's KDoc on why scope belongs to - // the declaration site (reference.ctelevelsup levels up from here), never the hop path. currentScopeStack = listOf(ownScope) + currentScopeStack.drop(reference.ctelevelsup) currentVar = bodyVar } diff --git a/generator/src/main/kotlin/norm/generator/NodeTreeProvenanceExpression.kt b/generator/src/main/kotlin/norm/generator/NodeTreeProvenanceExpression.kt index 0b1dcdd2..9780595f 100644 --- a/generator/src/main/kotlin/norm/generator/NodeTreeProvenanceExpression.kt +++ b/generator/src/main/kotlin/norm/generator/NodeTreeProvenanceExpression.kt @@ -5,35 +5,25 @@ package norm.generator * points at. * * [sql] must be the query's original text — never [nodeTreeText]'s sentinel-substituted or deparsed - * form — so a sentinel literal built only to satisfy `?`'s type during analysis can never leak into - * generated KDoc. [nodeTreeText] is trusted only for where the expression lives ([provenance]'s CTE - * name and body position); [sql] is trusted only for what it says. + * form. [nodeTreeText] is trusted only for where the expression lives ([provenance]'s CTE name and + * body position); [sql] is trusted only for what it says. * - * A text-only re-lex of the CTE body can mis-split or mis-merge an item boundary (an - * unbalanced-looking comment, a pathological string literal) without the item count changing, - * silently handing back a neighboring item's expression. Every gate below returns `null` (no - * provenance) instead of risking a wrong one: - * - the CTE [provenance] points at is found by replaying [NodeTreeColumnProvenance.hops] step by - * step (see [scopedNodeTreeCteQueryBlock] and [scopedSqlCteDefinition]), never by a flat, name-only - * search — so a nested `WITH` that shadows an outer CTE of the same name resolves against the exact - * declaration [NodeTreeProvenanceResolver] walked to, not merely a same-named one elsewhere. - * - [parseOutputItemsWithAlias] over that CTE's body must yield exactly as many top-level items as - * [nodeTreeText] has non-junk body target entries for that CTE. - * - every position's own name — not merely [provenance]'s — must fold-match that position's own - * `:resname`, in order, and those `:resname`s must be pairwise unique; otherwise a mis-split that - * happens to preserve the item count could pass by shifting text between two positions sharing a - * name. - * - [provenance]'s own item must have a verifiable name: an explicit `AS x`, an implicit trailing - * alias token, or being itself a bare column reference — see [verifiedItem]. - * - [provenance]'s own matched item must not have been verified only via an implicit alias: its - * complete text is legal only as a select-list item (`UPPER(name) y`), never as a standalone - * expression — the context a `@property` source reference renders it in — because whether a - * trailing bare word is an alias or a required operand (`ts AT TIME ZONE timezone`) cannot be - * decided from text alone. An explicit `AS` alias has no such problem: [extractAlias] already - * splits it off before the item's expression is formed. - * - the matched item must not be a bare column pass-through (see [isBareColumnPassThrough]) — a CTE - * body that merely forwards a column unchanged has nothing of its own to report, the same rule - * [TypeRepository.buildTypeProjectionForQuery] applies to a top-level plain column reference. + * Returns `null` instead of a possibly-wrong expression if any of these checks fails: + * - the CTE [provenance] points at is resolved by replaying [NodeTreeColumnProvenance.hops] step by + * step ([scopedNodeTreeCteQueryBlock], [scopedSqlCteDefinition]), so a nested `WITH` that shadows an + * outer CTE of the same name resolves against the exact declaration, not merely a same-named one + * elsewhere. + * - [parseOutputItemsWithAlias] over that CTE's body yields exactly as many top-level items as + * [nodeTreeText] has non-junk body target entries for that CTE, and every position's own `:resname` + * fold-matches, in order, with `:resname`s pairwise unique. + * - [provenance]'s own item has a verifiable name: an explicit `AS x`, an implicit trailing alias + * token, or being itself a bare column reference (see [verifiedItem]). + * - the matched item was not verified only via an implicit alias: whether a trailing bare word is an + * alias or a required operand cannot be decided from text alone (`ts AT TIME ZONE timezone`'s + * `:resname` is `timezone` whether or not `timezone` is really an alias), and the matched text is + * only ever legal as a select-list item, never as the standalone expression a `@property` source + * reference would render it in. + * - the matched item is not a bare column pass-through (see [isBareColumnPassThrough]). * * @param sql The query's original SQL text (containing the developer's own `?` placeholders, if * any) — never [nodeTreeText]'s sentinel-substituted or deparsed form. @@ -43,7 +33,7 @@ package norm.generator * called when it is non-`null`. * @param parser The node-tree parser to use; defaults to a fresh, stateless instance. * @return The CTE body's expression text, [collapseCosmeticWhitespace]-normalized and with any - * comment [stripComments]-removed, exactly as the developer wrote it — or `null` if any gate above + * comment [stripComments]-removed, exactly as the developer wrote it — or `null` if any check above * fails. */ internal fun resolveNodeTreeProvenanceExpression( @@ -72,13 +62,9 @@ internal fun resolveNodeTreeProvenanceExpression( if (provenance.bodyPosition < 1 || provenance.bodyPosition > items.size) return null val matchedItem = items[provenance.bodyPosition - 1] - // A bare column reference is a chained pass-through with nothing of its own to report -- see this - // function's own KDoc. if (isBareColumnPassThrough(matchedItem)) return null val matched = verifiedItems[provenance.bodyPosition - 1] - // The matched item's complete text (including its own implicit alias) is only ever a legal - // select-list item, never a legal standalone expression -- see this function's own KDoc. if (matched.matchKind == AliasMatchKind.IMPLICIT_ALIAS) return null return collapseCosmeticWhitespace(stripComments(matched.expression)) @@ -89,12 +75,10 @@ internal fun resolveNodeTreeProvenanceExpression( * explicit `AS` alias on a plain column, or the same shape spelled with an implicit alias * (`description dx`) — in which case there is nothing of [item]'s own to report as "the expression". * - * The implicit-alias branch here is the one place [splitTrailingImplicitAlias]'s guessed split - * decides something about [item], rather than merely verifying a `:resname` match (see - * [verifiedItem]). Safe because a wrong guess only makes this function more conservative (reporting - * a pass-through that wasn't one, so the caller reports nothing) or less conservative (missing a - * real pass-through, which then falls through to [verifiedItem]'s own [AliasMatchKind.IMPLICIT_ALIAS] - * decline instead) — never a wrong value emitted. + * A wrong guess from [splitTrailingImplicitAlias] here only makes this function more conservative + * (reporting a pass-through that wasn't one) or less conservative (missing a real one, which then + * falls through to [verifiedItem]'s own [AliasMatchKind.IMPLICIT_ALIAS] decline) — never a wrong + * value emitted. */ private fun isBareColumnPassThrough(item: OutputItemWithAlias): Boolean { if (item.selectItem.columnName != null) return true @@ -108,13 +92,12 @@ private fun isBareColumnPassThrough(item: OutputItemWithAlias): Boolean { * against its `:resname`. [resolveNodeTreeProvenanceExpression] emits the expression as-is for * [EXPLICIT_ALIAS]/[BARE_COLUMN_REFERENCE] (the latter unreachable there in practice, since * [isBareColumnPassThrough] already declines a bare column reference first), but declines - * [IMPLICIT_ALIAS] outright — see that function's own KDoc for why. + * [IMPLICIT_ALIAS] outright. */ private enum class AliasMatchKind { EXPLICIT_ALIAS, BARE_COLUMN_REFERENCE, IMPLICIT_ALIAS } /** - * @property expression The item's complete, uncut text — see [resolveNodeTreeProvenanceExpression] - * for why cutting is never attempted. + * @property expression The item's complete, uncut text. * @property matchKind Which naming rule verified [expression] against its `:resname`. */ private data class VerifiedItem(val expression: String, val matchKind: AliasMatchKind) @@ -128,11 +111,9 @@ private data class VerifiedItem(val expression: String, val matchKind: AliasMatc * [OutputItemWithAlias.alias] still carries its own quotes, if any. * 2. A bare column reference with no alias at all — PostgreSQL exposes such an item under the * column's own name. - * 3. Neither of the above: the item's text may still end with an implicit (no-`AS`) alias token that - * happens to be the same string PostgreSQL reports for a keyword-operand shape's own trailing - * operand (`ts AT TIME ZONE timezone`'s `:resname` is `timezone`, whether or not `timezone` is - * really an alias). [splitTrailingImplicitAlias] is used only to find that trailing token and - * fold-compare it against [resultName]; the split-off expression half of its result is never read. + * 3. Neither of the above: the item's text may still end with an implicit (no-`AS`) alias token + * ([splitTrailingImplicitAlias]) fold-compared against [resultName]; the split-off expression half + * of its result is never read. */ private fun verifiedItem(item: OutputItemWithAlias, resultName: String): VerifiedItem? { val alias = item.alias @@ -157,19 +138,16 @@ private fun verifiedItem(item: OutputItemWithAlias, resultName: String): Verifie } /** - * Replays [hops] against [nodeTreeText]'s own `:cteList` structure, maintaining the same scope-stack - * shape [NodeTreeProvenanceResolver.resolveVar] built while producing [hops], and returns the last - * hop's CTE body (`:ctequery` block) — the exact declaration [NodeTreeProvenanceResolver] resolved - * against, never merely a same-named one elsewhere in the tree. + * Replays [hops] against [nodeTreeText]'s own `:cteList` structure and returns the last hop's CTE + * body (`:ctequery` block) — the exact declaration [NodeTreeProvenanceResolver] resolved against, + * never merely a same-named one elsewhere in the tree. * - * Mirrors [NodeTreeProvenanceResolver.resolveVar]'s scope-stack bookkeeping exactly: entering a hop's - * CTE body pushes that body's own `:cteList` onto `scopeStack.drop(hop.ctelevelsup)`, not the full - * accumulated stack, because a hop into a sibling CTE is not a nesting level. + * Mirrors [NodeTreeProvenanceResolver.resolveVar]'s scope-stack bookkeeping: entering a hop's CTE body + * pushes that body's own `:cteList` onto `scopeStack.drop(hop.ctelevelsup)`, not the full accumulated + * stack, because a hop into a sibling CTE is not a nesting level. * * @return `null` if any hop's [CteHop.ctelevelsup] addresses a scope-stack depth that does not exist, - * or its [CteHop.name] is not declared in that scope's `:cteList` — neither can happen for [hops] - * genuinely produced against this same [nodeTreeText], but the caller still treats either as - * "cannot resolve" rather than assume it. + * or its [CteHop.name] is not declared in that scope's `:cteList`. */ private fun scopedNodeTreeCteQueryBlock(nodeTreeText: String, hops: List, parser: PgNodeTreeParser): String? { var scopeStack = listOf(parser.parseCteList(nodeTreeText).associateBy { it.name }) @@ -190,14 +168,10 @@ private fun scopedNodeTreeCteQueryBlock(nodeTreeText: String, hops: List * returns the last hop's [CteDefinition]. * * Every returned (and intermediate) [CteDefinition]'s [CteDefinition.bodyOpenParenthesis]/ - * [CteDefinition.bodyCloseParenthesis] are re-based to index into [sql] itself — never the (possibly - * nested) body substring a definition was found in — since [resolveNodeTreeProvenanceExpression] + * [CteDefinition.bodyCloseParenthesis] are re-based to index into [sql] itself, never the (possibly + * nested) body substring a definition was found in, since [resolveNodeTreeProvenanceExpression] * slices [sql] directly by whichever [CteDefinition] this function returns. * - * A single `WITH` clause can never legally declare the same name twice (PostgreSQL rejects that at - * parse time), so lookup within one scope level never needs a uniqueness check: the scope stack - * itself supplies the disambiguation a flat, whole-statement name search cannot. - * * @return `null` if any hop's [CteHop.ctelevelsup] addresses a scope-stack depth that does not exist, * or its [CteHop.name] does not fold-match ([foldIdentifier] against [CteDefinition.rawName]) any * definition in that scope @@ -213,8 +187,6 @@ private fun scopedSqlCteDefinition(sql: String, hops: List): CteDefiniti resolvedDefinition = definition val bodyOffset = definition.bodyOpenParenthesis + 1 val bodyText = sql.substring(bodyOffset, definition.bodyCloseParenthesis) - // drop, not prepend-onto-the-full-stack: see scopedNodeTreeCteQueryBlock's KDoc, which this - // mirrors exactly. scopeStack = listOf(rebasedCteDefinitions(bodyText, bodyOffset)) + scopeStack.drop(hop.ctelevelsup) } return resolvedDefinition @@ -223,8 +195,7 @@ private fun scopedSqlCteDefinition(sql: String, hops: List): CteDefiniti /** * [parseCteClause]'s own definitions for [text], with every [CteDefinition.bodyOpenParenthesis]/ * [CteDefinition.bodyCloseParenthesis] shifted by [offset] so they index into the original SQL - * string [text] was sliced from, rather than [text] itself. See [scopedSqlCteDefinition] for why - * every level of its scope stack needs definitions rebased this way. + * string [text] was sliced from, rather than [text] itself. */ private fun rebasedCteDefinitions(text: String, offset: Int): List = parseCteClause(text)?.definitions?.map { diff --git a/generator/src/main/kotlin/norm/generator/NullabilityCatalog.kt b/generator/src/main/kotlin/norm/generator/NullabilityCatalog.kt new file mode 100644 index 00000000..c47edc2a --- /dev/null +++ b/generator/src/main/kotlin/norm/generator/NullabilityCatalog.kt @@ -0,0 +1,469 @@ +package norm.generator + +import java.sql.Connection + +/** + * Loads the function-strictness, function-safe-list, and column-nullability facts from + * PostgreSQL's system catalogs that [ColumnNullabilityAnalyzer] needs to answer "is this specific + * query's result column nullable" — every fact is a lazy, cached read keyed by OID or + * `(relid, attnum)`, computed once per instance and reused for its lifetime. + * + * @param connection An open JDBC connection to a PostgreSQL database with the schema applied. + */ +internal class NullabilityCatalog(private val connection: Connection) { + + /** + * Maps function OIDs to their strictness flag from `pg_proc.proisstrict`. + * + * A strict function returns `null` when any argument is `null` — useful for determining + * expression nullability from the node tree. Keyed by OID for direct lookup from + * FUNCEXPR/OPEXPR/WINDOWFUNC nodes. Includes regular functions (`prokind = 'f'`) and + * window functions (`prokind = 'w'`). Excludes aggregates (`prokind = 'a'`) — those + * use [aggregateHasNonNullInitialValue] instead. + */ + val functionStrictnessByOid: Map by lazy(::loadFunctionStrictness) + + /** + * OIDs of IMMUTABLE, non-set-returning functions (`pg_proc.provolatile = 'i' AND NOT proretset`). + * + * Also covers operators, with no separate `pg_operator` lookup needed: [PgNodeExpression.OpExpr] + * and [PgNodeExpression.ScalarArrayOpExpr] are keyed by `opfuncid`/`oprcode` — the operator's + * *implementing function* OID — which is itself a `pg_proc` row already captured by this single + * query, unlike [neverNullForNonNullInputOids], which needs its own `pg_operator` query because it + * safe-lists specific (symbol, operand types) triples rather than a volatility flag every + * `pg_proc` row already carries. + * + * Used by [NodeTreeNullabilityAnalyzer.isSafeFromGroupingSetNullExtension]'s `foldsToConst` leg — + * see that method's KDoc for why IMMUTABLE specifically (not STRICT, and not restricted to + * `pg_catalog`) is the correct test: this mirrors PostgreSQL's own planner rule for constant + * folding directly, rather than an empirically-swept safe-list, so a user-defined `IMMUTABLE` + * function is exactly as fold-safe as a built-in one and no namespace restriction is needed. + */ + val immutableFunctionOids: Set by lazy(::loadImmutableFunctionOids) + + /** + * Maps aggregate function OIDs to whether they have a non-null initial transition value. + * + * Aggregates with non-null `agginitval` (like COUNT with `agginitval = '0'`) return a + * non-null value for empty groups. Aggregates with `null` `agginitval` (SUM, AVG, MIN, MAX) + * return `null` for empty groups. + * + * Returns `null` for absent keys — this can occur if the OID belongs to a non-aggregate function. + */ + val aggregateHasNonNullInitialValue: Map by lazy(::loadAggregateInitialValues) + + /** + * OIDs of non-strict functions that are guaranteed to never return `null` for any combination of + * argument values passed in the ordinary (non-`VARIADIC`) calling form, including when every + * argument is `null`. Currently `concat` only: `concat(NULL::text, NULL::text)` returns `''` + * (empty string), never `null`. + * + * The `VARIADIC` calling form (`concat(VARIADIC arr)`) is a different case this list's claim does + * not cover: it passes the array argument itself as one value rather than exploding it into + * elements, and `concat(VARIADIC arr)` is `null` when `arr` itself is `null` (PostgreSQL 16-18). + * [PgNodeExpression.FuncExpr.isVariadic] exists specifically so [NodeTreeNullabilityAnalyzer.isNonNull] + * and [NodeTreeNullabilityAnalyzer.isSafeFromGroupingSetNullExtension] can detect this form and + * require every argument non-null instead of trusting this list unconditionally — see both + * methods' KDoc. + * + * `concat_ws` is not on this list at all, even for the ordinary calling form, despite also being + * non-strict: it is non-null only when its first argument (the separator) is non-null — + * `concat_ws(NULL, 'x', 'y')` returns `null` (PostgreSQL 16-18), because a `null` separator + * poisons the whole result even though the later arguments are individually null-tolerant. That + * argument-position-dependent condition does not fit "unconditionally non-null", so it is modeled + * separately — see [nonNullIffFirstArgumentNonNullFunctionOids] and + * [NodeTreeNullabilityAnalyzer]'s `concat_ws` handling in `isNonNull`'s `FuncExpr` branch. + * + * [NodeTreeNullabilityAnalyzer.isSafeFromGroupingSetNullExtension] treats membership on this list + * (for a non-`VARIADIC` call) as an unconditional safety proof for the grouping-sets + * null-extension gate specifically because "non-null regardless of input" also means "non-null + * regardless of which argument grouping-set null-extension replaces with `null`". A function that + * is only non-null for a particular argument (like `concat_ws`'s separator) does not have that + * property — null-extension could target exactly that argument — so it must never be added here. + * + * Restricted to `pronamespace = 'pg_catalog'` at query time — a user-defined function sharing the + * name `concat` must not ride along onto this list; see the loader. + */ + val alwaysNonNullFunctionOids: Set by lazy(::loadAlwaysNonNullFunctions) + + /** + * OIDs of functions that are non-null if and only if their first argument is non-null, regardless + * of any other argument's nullability, in the ordinary (non-`VARIADIC`) calling form. Currently + * `concat_ws` only: `concat_ws(',', NULL, NULL)` returns `','`-joined empty string (`''`, + * non-null) but `concat_ws(NULL, 'x', 'y')` returns `null` — the separator (first argument) alone + * determines whether the whole call can be `null`. + * + * The `VARIADIC` calling form (`concat_ws(',', VARIADIC arr)`) does not get this treatment: it + * passes the array argument itself as one value, and `concat_ws(',', VARIADIC arr)` is `null` + * when `arr` itself is `null` even though the literal separator is non-null (PostgreSQL 16-18) — + * see [PgNodeExpression.FuncExpr.isVariadic]'s KDoc. + * + * Used by [NodeTreeNullabilityAnalyzer.isNonNull]'s [PgNodeExpression.FuncExpr] branch (for the + * non-`VARIADIC` form only). Not used by the grouping-sets safety gate + * ([NodeTreeNullabilityAnalyzer.isSafeFromGroupingSetNullExtension]) at all, `VARIADIC` or not: + * unlike [alwaysNonNullFunctionOids], this property depends on which argument is non-null, so a + * `Var` in the first-argument position is exactly as unsafe under grouping-set null-extension as + * any other `Var` — the generic aggregate/window-domination rule already handles it correctly + * without a dedicated leg. + * + * Restricted to `pronamespace = 'pg_catalog'` at query time — a user-defined function sharing the + * name `concat_ws` must not ride along onto this list; see the loader. + */ + val nonNullIffFirstArgumentNonNullFunctionOids: Set by lazy(::loadNonNullIffFirstArgumentNonNullFunctionOids) + + /** + * OIDs of functions, cast functions, and operators (materialized to their implementing function + * OID via `pg_operator.oprcode`) that are proven total on non-null input — every combination of + * non-null arguments produces a non-null result. An error is fine; only a silent `null` return + * disqualifies a candidate. + * + * This is verified only for the ordinary, element-wise calling convention (see `SafeListSweepTest`). + * [NodeTreeNullabilityAnalyzer.isNonNull]'s [PgNodeExpression.FuncExpr] branch never consults + * this set for a `VARIADIC` call: a non-null array argument says nothing about whether an + * element inside it is non-null, and no function on this list is variadic today (`provariadic <> + * 0` intersected with every safe-listed name here is empty on PostgreSQL 16-18) — but this must + * not silently start trusting the list for that shape the moment one is added. See + * [NodeTreeNullabilityAnalyzer]'s `isNeverNullForNonNullInput` KDoc. + * + * `pg_proc.proisstrict` is not sufficient for this on its own. Strict only guarantees + * NULL-in => NULL-out; it says nothing about the converse. `substring(text, '(z)')` (regex, no + * match), `regexp_match(text, pattern)` (no match), and `array_length(ARRAY[]::text[], 1)` + * (empty array) are all strict and all return `null` on fully non-null, well-typed input. Any + * inference rule built from strictness alone is therefore unsound. This set exists to be an + * additional conjunct alongside strictness in [NodeTreeNullabilityAnalyzer], never a + * replacement for it — so an unforeseen non-strict overload of a listed name can never slip + * through. + * + * Functions are safe-listed by `pg_proc.proname` plus argument type signature — see + * [NeverNullSafeLists.NEVER_NULL_FUNCTION_SIGNATURES] — restricted to `pronamespace = 'pg_catalog'`. Keying by + * name alone is not safe: `lower(anyrange)`/`upper(anyrange)`/`lower(anymultirange)`/ + * `upper(anymultirange)` share `proname` with the totally-safe `lower(text)`/`upper(text)` but + * return `null` on a non-null, well-typed, non-empty-but-unbounded range or an empty range — + * `SELECT upper(int4range '[1,)')` and `SELECT lower(int4range 'empty')` both return `null`. + * `substring` is the reason a signature-only match still is not always enough on its own: + * `substring(text, int, int)` is total but `substring(text FROM pattern)` is not, and both + * would share the same two-argument-count shape if only argument count were checked — this is + * why the match is on the full ordered list of argument type names (via `pg_type.typname`), not + * just arity. `substring` itself is simply left off the list entirely rather than enumerated, + * since its regex overloads are non-total. + * + * Casts are safe-listed by (source type, target type) pair — see + * [NeverNullSafeLists.NEVER_NULL_CAST_SIGNATURES] — rather than a class-wide blanket over every + * `pg_cast.castfunc` in `pg_catalog`. A blanket was + * tried first and is false: `('null'::jsonb)::int4` (and every other `jsonb` → numeric/`boolean` + * cast) returns `null` on well-typed, non-null input with no error, because the cast function + * special-cases the JSON literal `null` rather than raising "cannot convert". A sweep of every + * `jsonb`-targeting numeric/`boolean` cast confirmed this for all seven overloads (`int2`, `int4`, + * `int8`, `numeric`, `float4`, `float8`, `bool`); none of the seven appear in + * [NeverNullSafeLists.NEVER_NULL_CAST_SIGNATURES]. The same sweep also found `timestamp`/`timestamptz` → + * `time`/`timetz` silently returns `null` for the infinite (`'infinity'`/`'-infinity'`) input, + * rather than erroring the way `'infinity'::interval::time` does — so those three pairs are + * excluded too. Every other pair the sweep checked (see [SafeListSweepTest] for the corpus and the + * full case count) proved total, including on `NaN`, `Infinity`, `-Infinity`, min/max integer + * values, and empty strings. + * + * pgcrypto's `digest` and `hmac` are the one extension carve-out, keyed through `pg_depend` + * (`deptype = 'e'`) to the `pgcrypto` extension itself, so a user-defined `digest` in `public` + * cannot ride this carve-out. `encode`/`decode` are ordinary `pg_catalog` functions and are + * safe-listed on the main function list above, not here. All four `digest`/`hmac` overloads + * (`digest(text, text)`, `digest(bytea, text)`, `hmac(text, text, text)`, `hmac(bytea, bytea, + * text)`) are total on empty non-null input; an unrecognized hash algorithm name errors rather + * than returning `null`. + * + * Operators are safe-listed by (symbol, left operand type, right operand type) triple — see + * [NeverNullSafeLists.NEVER_NULL_OPERATOR_SIGNATURES] — restricted to `oprnamespace = 'pg_catalog'`, and + * materialized to the OID of the implementing function via `oprcode`, the same OID space + * [PgNodeExpression.OpExpr] and [PgNodeExpression.ScalarArrayOpExpr] (e.g. `= ANY(...)`) are + * keyed by, so no separate operator-specific lookup is needed. Symbol alone is not safe: `path + + * path` (`path_add`) shares the `+` symbol with the totally-safe `int4 + int4`, but returns + * `null`, not an error, when either operand is a closed path (`SELECT ((0,0),(1,1),(2,0)) + + * ((0,0),(1,1),(2,0))` on two well-typed, non-null closed paths). A sweep of every + * symbol-restricted-but-unrestricted-by-type combination found exactly this one bad shape; `path` + * is entirely absent from [NeverNullSafeLists.NEVER_NULL_OPERATOR_SIGNATURES] as a + * result — every triple that remains was independently swept and found total (see + * [SafeListSweepTest]). A left or right type of `null` in a signature means the operator is + * unary on that side (no left operand for a prefix operator, no right operand for a postfix + * operator), mirroring `pg_operator.oprleft`/`oprright` themselves being `0` (no operand) for a + * unary operator — e.g. unary (prefix) `-` (negation), `+`, and `~` (bitwise complement) are all + * prefix-only overloads of symbols that are also binary elsewhere in this same list (binary `-` + * is subtraction, binary `~` is regex match); they needed adding here alongside the binary + * overloads because the earlier symbol-only blanket rule this list replaced made every overload — + * unary and binary alike — safe together, and losing the unary overloads would have been an + * unintended narrowing. + * + * Omitting a signature from this set only widens the result to nullable — it never narrows a + * truly nullable expression to non-null — so when in doubt about whether a specific signature is + * total on every non-null, well-typed input (including infinite/empty/unbounded edge values, not + * just "typical" ones — see [NeverNullSafeLists.NEVER_NULL_FUNCTION_SIGNATURES] for the `extract`/`date_part` + * counterexample this note exists to flag), the correct default is to leave it off. + */ + val neverNullForNonNullInputOids: Set by lazy(::loadNeverNullForNonNullInputOids) + + /** + * OIDs of the 3-argument overloads of `lag` and `lead` window functions. These overloads accept + * `(value, offset, default)` and return a non-null result when both the value expression and the + * default expression are non-null — the default fills in for rows at window boundaries where + * 1-arg `lag`/`lead` would return `null`. + */ + val lagLeadWithDefaultOids: Set by lazy(::loadLagLeadWithDefaultOids) + + /** + * Maps `(relid, attnum)` pairs to `pg_attribute.attnotnull`. + * + * Used by [NodeTreeNullabilityAnalyzer] to determine whether a source column (referenced + * by a VAR node) is declared NOT NULL in the schema. Only includes user-visible columns + * (`attnum > 0` and `NOT attisdropped`). + * + * Returns `null` for absent keys — this occurs for columns not present in `pg_attribute` + * (e.g., virtual columns, system columns with attnum <= 0). + */ + val columnNotNullByRelidAndAttnum: Map, Boolean> by lazy(::loadColumnNotNull) + + /** + * Maps `(relid, attnum)` pairs to `pg_attribute.attname` — every user-visible column of every + * relation kind (base table, view, materialized view; `attnum > 0` and `NOT attisdropped`), not + * just base tables. + * + * Used by [ColumnNullabilityAnalyzer] to resolve a `TargetEntry`'s `:resorigtbl`/`:resorigcol` + * back to the real source column name, for a result column whose own select-list item is merely a + * reference to an alias assigned somewhere upstream (a CTE's own `RETURNING`/`SELECT` list + * renaming a column, e.g.) — PostgreSQL's `markTargetListOrigins` walks through such a reference to + * find the ultimate source column, so this is a plain OID/attnum lookup, not a name-based one. + * + * Returns `null` for absent keys — expected for any entry `:resorigtbl 0`/`:resorigcol 0` + * represents (a computed expression, an aggregate, a set-operation branch, or a `USING`/`NATURAL` + * merged join column has no single source column at all). + */ + val columnNameByRelidAndAttnum: Map, String> by lazy(::loadColumnNameByRelidAndAttnum) + + /** + * Shared strictness lookup used by [ColumnNullabilityAnalyzer.buildAnalyzer]'s `isStrict` + * parameter and by [NodeTreeNullabilityAnalyzer.qualProvenNonNullVars]. + */ + internal val isStrictFunction: (Int) -> Boolean = { oid -> functionStrictnessByOid[oid] == true } + + private fun loadColumnNotNull(): Map, Boolean> = buildMap { + connection.createStatement().use { stmt -> + // No schema filter — relids come from the query's rtable and may reference tables + // from any schema. Filtering by schema here would miss tables in non-default schemas. + stmt.executeQuery( + """ + SELECT attrelid::integer, attnum, attnotnull + FROM pg_catalog.pg_attribute + WHERE attnum > 0 AND NOT attisdropped + """.trimIndent(), + ).use { rs -> + while (rs.next()) { + put(rs.getInt("attrelid") to rs.getInt("attnum"), rs.getBoolean("attnotnull")) + } + } + } + } + + private fun loadColumnNameByRelidAndAttnum(): Map, String> = buildMap { + connection.createStatement().use { stmt -> + // No schema filter, same reasoning as loadColumnNotNull above — a resorigtbl OID can name a + // relation in any schema. + stmt.executeQuery( + """ + SELECT attrelid::integer, attnum, attname + FROM pg_catalog.pg_attribute + WHERE attnum > 0 AND NOT attisdropped + """.trimIndent(), + ).use { rs -> + while (rs.next()) { + put(rs.getInt("attrelid") to rs.getInt("attnum"), rs.getString("attname")) + } + } + } + } + + private fun loadFunctionStrictness(): Map = buildMap { + connection.createStatement().use { stmt -> + // Include regular functions ('f') and window functions ('w') — both appear in node tree expressions. + // Excludes procedures ('p') and aggregates ('a') — aggregates use agginitval, not strictness. + stmt.executeQuery( + "SELECT oid::integer, proisstrict FROM pg_catalog.pg_proc WHERE prokind IN ('f', 'w')", + ).use { rs -> + while (rs.next()) { + put(rs.getInt("oid"), rs.getBoolean("proisstrict")) + } + } + } + } + + private fun loadImmutableFunctionOids(): Set = buildSet { + connection.createStatement().use { stmt -> + // Regular functions and window functions, matching loadFunctionStrictness's scope; excludes + // procedures ('p') and aggregates ('a'), which are never foldable subexpressions here. + // + // No separate pg_operator/oprcode query is needed: an operator's implementing function + // (oprcode) is itself a row in pg_proc, so this single query already covers operators too — + // PgNodeExpression.OpExpr/ScalarArrayOpExpr are keyed by that same function OID, not by any + // pg_operator-specific ID. No operator oprcode OID satisfying the volatility/proretset filter + // falls outside this query's result. + stmt.executeQuery( + "SELECT oid::integer FROM pg_catalog.pg_proc WHERE provolatile = 'i' AND NOT proretset AND prokind IN ('f', 'w')", + ).use { rs -> + while (rs.next()) add(rs.getInt("oid")) + } + } + } + + private fun loadAggregateInitialValues(): Map = buildMap { + connection.createStatement().use { stmt -> + // An aggregate is non-null for empty groups only when it has a non-null initial transition value + // AND no final function. Aggregates with a finalfunc (AVG, STDDEV, etc.) can return null even + // with a non-null agginitval because the finalfunc may produce null (e.g., AVG divides by zero + // count). + stmt.executeQuery( + "SELECT aggfnoid::integer, (agginitval IS NOT NULL AND aggfinalfn = 0) AS has_initial_value FROM pg_catalog.pg_aggregate", + ).use { rs -> + while (rs.next()) { + put(rs.getInt("aggfnoid"), rs.getBoolean("has_initial_value")) + } + } + } + } + + private fun loadAlwaysNonNullFunctions(): Set = buildSet { + connection.createStatement().use { stmt -> + // pronamespace restricted to pg_catalog: without this, a user-defined function named + // `concat` with different null behavior would ride onto this list by sharing the name. + stmt.executeQuery( + """ + SELECT p.oid::integer AS oid + FROM pg_catalog.pg_proc p + JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace + WHERE p.proname = 'concat' AND NOT p.proisstrict AND n.nspname = 'pg_catalog' + """.trimIndent(), + ).use { rs -> + while (rs.next()) add(rs.getInt("oid")) + } + } + } + + private fun loadNonNullIffFirstArgumentNonNullFunctionOids(): Set = buildSet { + connection.createStatement().use { stmt -> + // pronamespace restricted to pg_catalog — see loadAlwaysNonNullFunctions's identical guard. + stmt.executeQuery( + """ + SELECT p.oid::integer AS oid + FROM pg_catalog.pg_proc p + JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace + WHERE p.proname = 'concat_ws' AND NOT p.proisstrict AND n.nspname = 'pg_catalog' + """.trimIndent(), + ).use { rs -> + while (rs.next()) add(rs.getInt("oid")) + } + } + } + + private fun loadNeverNullForNonNullInputOids(): Set = buildSet { + connection.createStatement().use { stmt -> + val safeSignatureValues = NeverNullSafeLists.NEVER_NULL_FUNCTION_SIGNATURES.joinToString(", ") { signature -> + val nameLiteral = "'${signature.name}'" + val argumentTypesLiteral = if (signature.argumentTypeNames.isEmpty()) { + "ARRAY[]::text[]" + } else { + "ARRAY[${signature.argumentTypeNames.joinToString(", ") { typeName -> "'$typeName'" }}]" + } + "($nameLiteral, $argumentTypesLiteral)" + } + stmt.executeQuery( + """ + WITH safe_signature(proname, argument_types) AS ( + VALUES $safeSignatureValues + ) + SELECT p.oid::integer AS oid + FROM pg_catalog.pg_proc p + JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace + JOIN safe_signature s ON s.proname = p.proname + WHERE n.nspname = 'pg_catalog' + AND s.argument_types = COALESCE( + ( + SELECT array_agg(t.typname::text ORDER BY u.ordinality) + FROM unnest(p.proargtypes) WITH ORDINALITY AS u(type_oid, ordinality) + JOIN pg_catalog.pg_type t ON t.oid = u.type_oid + ), + ARRAY[]::text[] + ) + """.trimIndent(), + ).use { rs -> + while (rs.next()) add(rs.getInt("oid")) + } + } + connection.createStatement().use { stmt -> + val safeCastValues = NeverNullSafeLists.NEVER_NULL_CAST_SIGNATURES.joinToString(", ") { signature -> + "('${signature.sourceTypeName}', '${signature.targetTypeName}')" + } + stmt.executeQuery( + """ + WITH safe_cast(source_type, target_type) AS ( + VALUES $safeCastValues + ) + SELECT c.castfunc::integer AS oid + FROM pg_catalog.pg_cast c + JOIN pg_catalog.pg_type st ON st.oid = c.castsource + JOIN pg_catalog.pg_type tt ON tt.oid = c.casttarget + JOIN pg_catalog.pg_proc p ON p.oid = c.castfunc + JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace + JOIN safe_cast s ON s.source_type = st.typname AND s.target_type = tt.typname + WHERE c.castfunc != 0 AND n.nspname = 'pg_catalog' + """.trimIndent(), + ).use { rs -> + while (rs.next()) add(rs.getInt("oid")) + } + } + connection.createStatement().use { stmt -> + val safeOperatorValues = NeverNullSafeLists.NEVER_NULL_OPERATOR_SIGNATURES.joinToString(", ") { signature -> + val leftLiteral = signature.leftTypeName?.let { "'$it'" } ?: "NULL::text" + val rightLiteral = signature.rightTypeName?.let { "'$it'" } ?: "NULL::text" + "('${signature.symbol}', $leftLiteral, $rightLiteral)" + } + stmt.executeQuery( + """ + WITH safe_operator(symbol, left_type, right_type) AS ( + VALUES $safeOperatorValues + ) + SELECT o.oprcode::integer AS oid + FROM pg_catalog.pg_operator o + JOIN pg_catalog.pg_namespace n ON n.oid = o.oprnamespace + LEFT JOIN pg_catalog.pg_type lt ON lt.oid = o.oprleft + LEFT JOIN pg_catalog.pg_type rt ON rt.oid = o.oprright + JOIN safe_operator s ON s.symbol = o.oprname + AND s.left_type IS NOT DISTINCT FROM lt.typname + AND s.right_type IS NOT DISTINCT FROM rt.typname + WHERE n.nspname = 'pg_catalog' AND o.oprcode != 0 + """.trimIndent(), + ).use { rs -> + while (rs.next()) add(rs.getInt("oid")) + } + } + connection.createStatement().use { stmt -> + // pgcrypto extension carve-out, keyed through pg_depend so a user-defined `digest` or + // `hmac` outside the extension cannot ride along. + stmt.executeQuery( + """ + SELECT p.oid::integer AS oid FROM pg_catalog.pg_proc p + JOIN pg_catalog.pg_depend d + ON d.objid = p.oid AND d.classid = 'pg_catalog.pg_proc'::regclass AND d.deptype = 'e' + JOIN pg_catalog.pg_extension e ON e.oid = d.refobjid + WHERE e.extname = 'pgcrypto' AND p.proname IN ('digest', 'hmac') + """.trimIndent(), + ).use { rs -> + while (rs.next()) add(rs.getInt("oid")) + } + } + } + + private fun loadLagLeadWithDefaultOids(): Set = buildSet { + connection.createStatement().use { stmt -> + stmt.executeQuery( + "SELECT oid::integer FROM pg_catalog.pg_proc WHERE proname IN ('lag', 'lead') AND pronargs = 3 AND prokind = 'w'", + ).use { rs -> + while (rs.next()) add(rs.getInt("oid")) + } + } + } +} diff --git a/generator/src/main/kotlin/norm/generator/PgCatalogLoader.kt b/generator/src/main/kotlin/norm/generator/PgCatalogLoader.kt index 12ea3c6e..800c71cf 100644 --- a/generator/src/main/kotlin/norm/generator/PgCatalogLoader.kt +++ b/generator/src/main/kotlin/norm/generator/PgCatalogLoader.kt @@ -8,18 +8,20 @@ import java.sql.SQLException * Loads schema metadata from PostgreSQL's system catalogs via JDBC. * * Provides a lazy-loaded cache of [functionOverloads] computed once per instance, plus - * on-demand queries for enums, domains, column comments, and stored procedures. + * on-demand queries for enums, domains, column comments, and stored procedures. Function + * strictness, safe-list, and column-nullability facts live on the composed [NullabilityCatalog] + * instead, and per-query nullability analysis on the composed [ColumnNullabilityAnalyzer] — see + * each class's own KDoc for why its concern is split out from this one. * * @param connection An open JDBC connection to a PostgreSQL database with the schema applied. */ -internal class PgCatalogLoader(internal val connection: Connection) { - - internal val nodeTreeParser = PgNodeTreeParser() +internal class PgCatalogLoader(private val connection: Connection) { private val pgMajorVersion = connection.metaData.databaseMajorVersion - /** See [ColumnNullabilityAnalyzer]'s own KDoc for why this is a separate collaborator. */ - private val nullabilityAnalyzer = ColumnNullabilityAnalyzer(this) + private val nullabilityCatalog = NullabilityCatalog(connection) + + private val nullabilityAnalyzer = ColumnNullabilityAnalyzer(connection, nullabilityCatalog) init { checkPostgresVersion() @@ -36,272 +38,6 @@ internal class PgCatalogLoader(internal val connection: Connection) { */ val functionOverloads: Map> by lazy(::loadFunctionOverloads) - /** - * Maps function OIDs to their strictness flag from `pg_proc.proisstrict`. - * - * A strict function returns `null` when any argument is `null` — useful for determining - * expression nullability from the node tree. Keyed by OID for direct lookup from - * FUNCEXPR/OPEXPR/WINDOWFUNC nodes. Includes regular functions (`prokind = 'f'`) and - * window functions (`prokind = 'w'`). Excludes aggregates (`prokind = 'a'`) — those - * use [aggregateHasNonNullInitialValue] instead. - */ - val functionStrictnessByOid: Map by lazy(::loadFunctionStrictness) - - /** - * OIDs of IMMUTABLE, non-set-returning functions (`pg_proc.provolatile = 'i' AND NOT proretset`). - * - * Also covers operators, with no separate `pg_operator` lookup needed: [PgNodeExpression.OpExpr] - * and [PgNodeExpression.ScalarArrayOpExpr] are keyed by `opfuncid`/`oprcode` — the operator's - * *implementing function* OID — which is itself a `pg_proc` row already captured by this single - * query, unlike [neverNullForNonNullInputOids], which needs its own `pg_operator` query because it - * safe-lists specific (symbol, operand types) triples rather than a volatility flag every - * `pg_proc` row already carries. - * - * Used by [NodeTreeNullabilityAnalyzer.isSafeFromGroupingSetNullExtension]'s `foldsToConst` leg — - * see that method's KDoc for why IMMUTABLE specifically (not STRICT, and not restricted to - * `pg_catalog`) is the correct test: this mirrors PostgreSQL's own planner rule for constant - * folding directly, rather than an empirically-swept safe-list, so a user-defined `IMMUTABLE` - * function is exactly as fold-safe as a built-in one and no namespace restriction is needed. - */ - val immutableFunctionOids: Set by lazy(::loadImmutableFunctionOids) - - /** - * Maps aggregate function OIDs to whether they have a non-null initial transition value. - * - * Aggregates with non-null `agginitval` (like COUNT with `agginitval = '0'`) return a - * non-null value for empty groups. Aggregates with `null` `agginitval` (SUM, AVG, MIN, MAX) - * return `null` for empty groups. - * - * Returns `null` for absent keys — this can occur if the OID belongs to a non-aggregate function. - */ - val aggregateHasNonNullInitialValue: Map by lazy(::loadAggregateInitialValues) - - /** - * OIDs of non-strict functions that are guaranteed to never return `null` for any combination of - * argument values passed in the ordinary (non-`VARIADIC`) calling form, including when every - * argument is `null`. Currently `concat` only: `concat(NULL::text, NULL::text)` returns `''` - * (empty string), never `null`. - * - * The `VARIADIC` calling form (`concat(VARIADIC arr)`) is a different case this list's claim does - * not cover: it passes the array argument itself as one value rather than exploding it into - * elements, and `concat(VARIADIC arr)` is `null` when `arr` itself is `null` (PostgreSQL 16-18). - * [PgNodeExpression.FuncExpr.isVariadic] exists specifically so [NodeTreeNullabilityAnalyzer.isNonNull] - * and [NodeTreeNullabilityAnalyzer.isSafeFromGroupingSetNullExtension] can detect this form and - * require every argument non-null instead of trusting this list unconditionally — see both - * methods' KDoc. - * - * `concat_ws` is not on this list at all, even for the ordinary calling form, despite also being - * non-strict: it is non-null only when its first argument (the separator) is non-null — - * `concat_ws(NULL, 'x', 'y')` returns `null` (PostgreSQL 16-18), because a `null` separator - * poisons the whole result even though the later arguments are individually null-tolerant. That - * argument-position-dependent condition does not fit "unconditionally non-null", so it is modeled - * separately — see [nonNullIffFirstArgumentNonNullFunctionOids] and - * [NodeTreeNullabilityAnalyzer]'s `concat_ws` handling in `isNonNull`'s `FuncExpr` branch. - * - * [NodeTreeNullabilityAnalyzer.isSafeFromGroupingSetNullExtension] treats membership on this list - * (for a non-`VARIADIC` call) as an unconditional safety proof for the grouping-sets - * null-extension gate specifically because "non-null regardless of input" also means "non-null - * regardless of which argument grouping-set null-extension replaces with `null`". A function that - * is only non-null for a particular argument (like `concat_ws`'s separator) does not have that - * property — null-extension could target exactly that argument — so it must never be added here. - * - * Restricted to `pronamespace = 'pg_catalog'` at query time — a user-defined function sharing the - * name `concat` must not ride along onto this list; see the loader. - */ - val alwaysNonNullFunctionOids: Set by lazy(::loadAlwaysNonNullFunctions) - - /** - * OIDs of functions that are non-null if and only if their first argument is non-null, regardless - * of any other argument's nullability, in the ordinary (non-`VARIADIC`) calling form. Currently - * `concat_ws` only: `concat_ws(',', NULL, NULL)` returns `','`-joined empty string (`''`, - * non-null) but `concat_ws(NULL, 'x', 'y')` returns `null` — the separator (first argument) alone - * determines whether the whole call can be `null`. - * - * The `VARIADIC` calling form (`concat_ws(',', VARIADIC arr)`) does not get this treatment: it - * passes the array argument itself as one value, and `concat_ws(',', VARIADIC arr)` is `null` - * when `arr` itself is `null` even though the literal separator is non-null (PostgreSQL 16-18) — - * see [PgNodeExpression.FuncExpr.isVariadic]'s KDoc. - * - * Used by [NodeTreeNullabilityAnalyzer.isNonNull]'s [PgNodeExpression.FuncExpr] branch (for the - * non-`VARIADIC` form only). Not used by the grouping-sets safety gate - * ([NodeTreeNullabilityAnalyzer.isSafeFromGroupingSetNullExtension]) at all, `VARIADIC` or not: - * unlike [alwaysNonNullFunctionOids], this property depends on which argument is non-null, so a - * `Var` in the first-argument position is exactly as unsafe under grouping-set null-extension as - * any other `Var` — the generic aggregate/window-domination rule already handles it correctly - * without a dedicated leg. - * - * Restricted to `pronamespace = 'pg_catalog'` at query time — a user-defined function sharing the - * name `concat_ws` must not ride along onto this list; see the loader. - */ - val nonNullIffFirstArgumentNonNullFunctionOids: Set by lazy(::loadNonNullIffFirstArgumentNonNullFunctionOids) - - /** - * OIDs of functions, cast functions, and operators (materialized to their implementing function - * OID via `pg_operator.oprcode`) that are proven total on non-null input — every combination of - * non-null arguments produces a non-null result. An error is fine; only a silent `null` return - * disqualifies a candidate. - * - * This is verified only for the ordinary, element-wise calling convention (see `SafeListSweepTest`). - * [NodeTreeNullabilityAnalyzer.isNonNull]'s [PgNodeExpression.FuncExpr] branch never consults - * this set for a `VARIADIC` call: a non-null array argument says nothing about whether an - * element inside it is non-null, and no function on this list is variadic today (`provariadic <> - * 0` intersected with every safe-listed name here is empty on PostgreSQL 16-18) — but this must - * not silently start trusting the list for that shape the moment one is added. See - * [NodeTreeNullabilityAnalyzer]'s `isNeverNullForNonNullInput` KDoc. - * - * `pg_proc.proisstrict` is not sufficient for this on its own. Strict only guarantees - * NULL-in => NULL-out; it says nothing about the converse. `substring(text, '(z)')` (regex, no - * match), `regexp_match(text, pattern)` (no match), and `array_length(ARRAY[]::text[], 1)` - * (empty array) are all strict and all return `null` on fully non-null, well-typed input. Any - * inference rule built from strictness alone is therefore unsound. This set exists to be an - * additional conjunct alongside strictness in [NodeTreeNullabilityAnalyzer], never a - * replacement for it — so an unforeseen non-strict overload of a listed name can never slip - * through. - * - * Functions are safe-listed by `pg_proc.proname` plus argument type signature — see - * [NeverNullSafeLists.NEVER_NULL_FUNCTION_SIGNATURES] — restricted to `pronamespace = 'pg_catalog'`. Keying by - * name alone is not safe: `lower(anyrange)`/`upper(anyrange)`/`lower(anymultirange)`/ - * `upper(anymultirange)` share `proname` with the totally-safe `lower(text)`/`upper(text)` but - * return `null` on a non-null, well-typed, non-empty-but-unbounded range or an empty range — - * `SELECT upper(int4range '[1,)')` and `SELECT lower(int4range 'empty')` both return `null`. - * `substring` is the reason a signature-only match still is not always enough on its own: - * `substring(text, int, int)` is total but `substring(text FROM pattern)` is not, and both - * would share the same two-argument-count shape if only argument count were checked — this is - * why the match is on the full ordered list of argument type names (via `pg_type.typname`), not - * just arity. `substring` itself is simply left off the list entirely rather than enumerated, - * since its regex overloads are non-total. - * - * Casts are safe-listed by (source type, target type) pair — see - * [NeverNullSafeLists.NEVER_NULL_CAST_SIGNATURES] — rather than a class-wide blanket over every - * `pg_cast.castfunc` in `pg_catalog`. A blanket was - * tried first and is false: `('null'::jsonb)::int4` (and every other `jsonb` → numeric/`boolean` - * cast) returns `null` on well-typed, non-null input with no error, because the cast function - * special-cases the JSON literal `null` rather than raising "cannot convert". A sweep of every - * `jsonb`-targeting numeric/`boolean` cast confirmed this for all seven overloads (`int2`, `int4`, - * `int8`, `numeric`, `float4`, `float8`, `bool`); none of the seven appear in - * [NeverNullSafeLists.NEVER_NULL_CAST_SIGNATURES]. The same sweep also found `timestamp`/`timestamptz` → - * `time`/`timetz` silently returns `null` for the infinite (`'infinity'`/`'-infinity'`) input, - * rather than erroring the way `'infinity'::interval::time` does — so those three pairs are - * excluded too. Every other pair the sweep checked (see [SafeListSweepTest] for the corpus and the - * full case count) proved total, including on `NaN`, `Infinity`, `-Infinity`, min/max integer - * values, and empty strings. - * - * pgcrypto's `digest` and `hmac` are the one extension carve-out, keyed through `pg_depend` - * (`deptype = 'e'`) to the `pgcrypto` extension itself, so a user-defined `digest` in `public` - * cannot ride this carve-out. `encode`/`decode` are ordinary `pg_catalog` functions and are - * safe-listed on the main function list above, not here. All four `digest`/`hmac` overloads - * (`digest(text, text)`, `digest(bytea, text)`, `hmac(text, text, text)`, `hmac(bytea, bytea, - * text)`) are total on empty non-null input; an unrecognized hash algorithm name errors rather - * than returning `null`. - * - * Operators are safe-listed by (symbol, left operand type, right operand type) triple — see - * [NeverNullSafeLists.NEVER_NULL_OPERATOR_SIGNATURES] — restricted to `oprnamespace = 'pg_catalog'`, and - * materialized to the OID of the implementing function via `oprcode`, the same OID space - * [PgNodeExpression.OpExpr] and [PgNodeExpression.ScalarArrayOpExpr] (e.g. `= ANY(...)`) are - * keyed by, so no separate operator-specific lookup is needed. Symbol alone is not safe: `path + - * path` (`path_add`) shares the `+` symbol with the totally-safe `int4 + int4`, but returns - * `null`, not an error, when either operand is a closed path (`SELECT ((0,0),(1,1),(2,0)) + - * ((0,0),(1,1),(2,0))` on two well-typed, non-null closed paths). A sweep of every - * symbol-restricted-but-unrestricted-by-type combination found exactly this one bad shape; `path` - * is entirely absent from [NeverNullSafeLists.NEVER_NULL_OPERATOR_SIGNATURES] as a - * result — every triple that remains was independently swept and found total (see - * [SafeListSweepTest]). A left or right type of `null` in a signature means the operator is - * unary on that side (no left operand for a prefix operator, no right operand for a postfix - * operator), mirroring `pg_operator.oprleft`/`oprright` themselves being `0` (no operand) for a - * unary operator — e.g. unary (prefix) `-` (negation), `+`, and `~` (bitwise complement) are all - * prefix-only overloads of symbols that are also binary elsewhere in this same list (binary `-` - * is subtraction, binary `~` is regex match); they needed adding here alongside the binary - * overloads because the earlier symbol-only blanket rule this list replaced made every overload — - * unary and binary alike — safe together, and losing the unary overloads would have been an - * unintended narrowing. - * - * Omitting a signature from this set only widens the result to nullable — it never narrows a - * truly nullable expression to non-null — so when in doubt about whether a specific signature is - * total on every non-null, well-typed input (including infinite/empty/unbounded edge values, not - * just "typical" ones — see [NeverNullSafeLists.NEVER_NULL_FUNCTION_SIGNATURES] for the `extract`/`date_part` - * counterexample this note exists to flag), the correct default is to leave it off. - */ - val neverNullForNonNullInputOids: Set by lazy(::loadNeverNullForNonNullInputOids) - - /** - * OIDs of the 3-argument overloads of `lag` and `lead` window functions. These overloads accept - * `(value, offset, default)` and return a non-null result when both the value expression and the - * default expression are non-null — the default fills in for rows at window boundaries where - * 1-arg `lag`/`lead` would return `null`. - */ - val lagLeadWithDefaultOids: Set by lazy(::loadLagLeadWithDefaultOids) - - /** - * Maps `(relid, attnum)` pairs to `pg_attribute.attnotnull`. - * - * Used by [NodeTreeNullabilityAnalyzer] to determine whether a source column (referenced - * by a VAR node) is declared NOT NULL in the schema. Only includes user-visible columns - * (`attnum > 0` and `NOT attisdropped`). - * - * Returns `null` for absent keys — this occurs for columns not present in `pg_attribute` - * (e.g., virtual columns, system columns with attnum <= 0). - */ - val columnNotNullByRelidAndAttnum: Map, Boolean> by lazy(::loadColumnNotNull) - - /** - * Maps `(relid, attnum)` pairs to `pg_attribute.attname` — every user-visible column of every - * relation kind (base table, view, materialized view; `attnum > 0` and `NOT attisdropped`), not - * just base tables. - * - * Used by [ColumnNullabilityAnalyzer] to resolve a `TargetEntry`'s `:resorigtbl`/`:resorigcol` - * back to the real source column name, for a result column whose own select-list item is merely a - * reference to an alias assigned somewhere upstream (a CTE's own `RETURNING`/`SELECT` list - * renaming a column, e.g.) — PostgreSQL's `markTargetListOrigins` walks through such a reference to - * find the ultimate source column, so this is a plain OID/attnum lookup, not a name-based one. - * - * Returns `null` for absent keys — expected for any entry `:resorigtbl 0`/`:resorigcol 0` - * represents (a computed expression, an aggregate, a set-operation branch, or a `USING`/`NATURAL` - * merged join column has no single source column at all). - */ - val columnNameByRelidAndAttnum: Map, String> by lazy(::loadColumnNameByRelidAndAttnum) - - /** - * Shared strictness lookup used by every [buildAnalyzer] call site and by - * [NodeTreeNullabilityAnalyzer.qualProvenNonNullVars]. - */ - internal val isStrictFunction: (Int) -> Boolean = { oid -> functionStrictnessByOid[oid] == true } - - private fun loadColumnNotNull(): Map, Boolean> = buildMap { - connection.createStatement().use { stmt -> - // No schema filter — relids come from the query's rtable and may reference tables - // from any schema. Filtering by schema here would miss tables in non-default schemas. - stmt.executeQuery( - """ - SELECT attrelid::integer, attnum, attnotnull - FROM pg_catalog.pg_attribute - WHERE attnum > 0 AND NOT attisdropped - """.trimIndent(), - ).use { rs -> - while (rs.next()) { - put(rs.getInt("attrelid") to rs.getInt("attnum"), rs.getBoolean("attnotnull")) - } - } - } - } - - private fun loadColumnNameByRelidAndAttnum(): Map, String> = buildMap { - connection.createStatement().use { stmt -> - // No schema filter, same reasoning as loadColumnNotNull above — a resorigtbl OID can name a - // relation in any schema. - stmt.executeQuery( - """ - SELECT attrelid::integer, attnum, attname - FROM pg_catalog.pg_attribute - WHERE attnum > 0 AND NOT attisdropped - """.trimIndent(), - ).use { rs -> - while (rs.next()) { - put(rs.getInt("attrelid") to rs.getInt("attnum"), rs.getString("attname")) - } - } - } - } - private fun loadFunctionOverloads(): Map> = buildMap> { connection.createStatement().use { stmt -> @@ -330,195 +66,6 @@ internal class PgCatalogLoader(internal val connection: Connection) { } } - private fun loadFunctionStrictness(): Map = buildMap { - connection.createStatement().use { stmt -> - // Include regular functions ('f') and window functions ('w') — both appear in node tree expressions. - // Excludes procedures ('p') and aggregates ('a') — aggregates use agginitval, not strictness. - stmt.executeQuery( - "SELECT oid::integer, proisstrict FROM pg_catalog.pg_proc WHERE prokind IN ('f', 'w')", - ).use { rs -> - while (rs.next()) { - put(rs.getInt("oid"), rs.getBoolean("proisstrict")) - } - } - } - } - - private fun loadImmutableFunctionOids(): Set = buildSet { - connection.createStatement().use { stmt -> - // Regular functions and window functions, matching loadFunctionStrictness's scope; excludes - // procedures ('p') and aggregates ('a'), which are never foldable subexpressions here. - // - // No separate pg_operator/oprcode query is needed: an operator's implementing function - // (oprcode) is itself a row in pg_proc, so this single query already covers operators too — - // PgNodeExpression.OpExpr/ScalarArrayOpExpr are keyed by that same function OID, not by any - // pg_operator-specific ID. No operator oprcode OID satisfying the volatility/proretset filter - // falls outside this query's result. - stmt.executeQuery( - "SELECT oid::integer FROM pg_catalog.pg_proc WHERE provolatile = 'i' AND NOT proretset AND prokind IN ('f', 'w')", - ).use { rs -> - while (rs.next()) add(rs.getInt("oid")) - } - } - } - - private fun loadAggregateInitialValues(): Map = buildMap { - connection.createStatement().use { stmt -> - // An aggregate is non-null for empty groups only when it has a non-null initial transition value - // AND no final function. Aggregates with a finalfunc (AVG, STDDEV, etc.) can return null even - // with a non-null agginitval because the finalfunc may produce null (e.g., AVG divides by zero - // count). - stmt.executeQuery( - "SELECT aggfnoid::integer, (agginitval IS NOT NULL AND aggfinalfn = 0) AS has_initial_value FROM pg_catalog.pg_aggregate", - ).use { rs -> - while (rs.next()) { - put(rs.getInt("aggfnoid"), rs.getBoolean("has_initial_value")) - } - } - } - } - - private fun loadAlwaysNonNullFunctions(): Set = buildSet { - connection.createStatement().use { stmt -> - // pronamespace restricted to pg_catalog: without this, a user-defined function named - // `concat` with different null behavior would ride onto this list by sharing the name. - stmt.executeQuery( - """ - SELECT p.oid::integer AS oid - FROM pg_catalog.pg_proc p - JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace - WHERE p.proname = 'concat' AND NOT p.proisstrict AND n.nspname = 'pg_catalog' - """.trimIndent(), - ).use { rs -> - while (rs.next()) add(rs.getInt("oid")) - } - } - } - - private fun loadNonNullIffFirstArgumentNonNullFunctionOids(): Set = buildSet { - connection.createStatement().use { stmt -> - // pronamespace restricted to pg_catalog — see loadAlwaysNonNullFunctions's identical guard. - stmt.executeQuery( - """ - SELECT p.oid::integer AS oid - FROM pg_catalog.pg_proc p - JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace - WHERE p.proname = 'concat_ws' AND NOT p.proisstrict AND n.nspname = 'pg_catalog' - """.trimIndent(), - ).use { rs -> - while (rs.next()) add(rs.getInt("oid")) - } - } - } - - private fun loadNeverNullForNonNullInputOids(): Set = buildSet { - connection.createStatement().use { stmt -> - val safeSignatureValues = NeverNullSafeLists.NEVER_NULL_FUNCTION_SIGNATURES.joinToString(", ") { signature -> - val nameLiteral = "'${signature.name}'" - val argumentTypesLiteral = if (signature.argumentTypeNames.isEmpty()) { - "ARRAY[]::text[]" - } else { - "ARRAY[${signature.argumentTypeNames.joinToString(", ") { typeName -> "'$typeName'" }}]" - } - "($nameLiteral, $argumentTypesLiteral)" - } - stmt.executeQuery( - """ - WITH safe_signature(proname, argument_types) AS ( - VALUES $safeSignatureValues - ) - SELECT p.oid::integer AS oid - FROM pg_catalog.pg_proc p - JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace - JOIN safe_signature s ON s.proname = p.proname - WHERE n.nspname = 'pg_catalog' - AND s.argument_types = COALESCE( - ( - SELECT array_agg(t.typname::text ORDER BY u.ordinality) - FROM unnest(p.proargtypes) WITH ORDINALITY AS u(type_oid, ordinality) - JOIN pg_catalog.pg_type t ON t.oid = u.type_oid - ), - ARRAY[]::text[] - ) - """.trimIndent(), - ).use { rs -> - while (rs.next()) add(rs.getInt("oid")) - } - } - connection.createStatement().use { stmt -> - val safeCastValues = NeverNullSafeLists.NEVER_NULL_CAST_SIGNATURES.joinToString(", ") { signature -> - "('${signature.sourceTypeName}', '${signature.targetTypeName}')" - } - stmt.executeQuery( - """ - WITH safe_cast(source_type, target_type) AS ( - VALUES $safeCastValues - ) - SELECT c.castfunc::integer AS oid - FROM pg_catalog.pg_cast c - JOIN pg_catalog.pg_type st ON st.oid = c.castsource - JOIN pg_catalog.pg_type tt ON tt.oid = c.casttarget - JOIN pg_catalog.pg_proc p ON p.oid = c.castfunc - JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace - JOIN safe_cast s ON s.source_type = st.typname AND s.target_type = tt.typname - WHERE c.castfunc != 0 AND n.nspname = 'pg_catalog' - """.trimIndent(), - ).use { rs -> - while (rs.next()) add(rs.getInt("oid")) - } - } - connection.createStatement().use { stmt -> - val safeOperatorValues = NeverNullSafeLists.NEVER_NULL_OPERATOR_SIGNATURES.joinToString(", ") { signature -> - val leftLiteral = signature.leftTypeName?.let { "'$it'" } ?: "NULL::text" - val rightLiteral = signature.rightTypeName?.let { "'$it'" } ?: "NULL::text" - "('${signature.symbol}', $leftLiteral, $rightLiteral)" - } - stmt.executeQuery( - """ - WITH safe_operator(symbol, left_type, right_type) AS ( - VALUES $safeOperatorValues - ) - SELECT o.oprcode::integer AS oid - FROM pg_catalog.pg_operator o - JOIN pg_catalog.pg_namespace n ON n.oid = o.oprnamespace - LEFT JOIN pg_catalog.pg_type lt ON lt.oid = o.oprleft - LEFT JOIN pg_catalog.pg_type rt ON rt.oid = o.oprright - JOIN safe_operator s ON s.symbol = o.oprname - AND s.left_type IS NOT DISTINCT FROM lt.typname - AND s.right_type IS NOT DISTINCT FROM rt.typname - WHERE n.nspname = 'pg_catalog' AND o.oprcode != 0 - """.trimIndent(), - ).use { rs -> - while (rs.next()) add(rs.getInt("oid")) - } - } - connection.createStatement().use { stmt -> - // pgcrypto extension carve-out, keyed through pg_depend so a user-defined `digest` or - // `hmac` outside the extension cannot ride along. - stmt.executeQuery( - """ - SELECT p.oid::integer AS oid FROM pg_catalog.pg_proc p - JOIN pg_catalog.pg_depend d - ON d.objid = p.oid AND d.classid = 'pg_catalog.pg_proc'::regclass AND d.deptype = 'e' - JOIN pg_catalog.pg_extension e ON e.oid = d.refobjid - WHERE e.extname = 'pgcrypto' AND p.proname IN ('digest', 'hmac') - """.trimIndent(), - ).use { rs -> - while (rs.next()) add(rs.getInt("oid")) - } - } - } - - private fun loadLagLeadWithDefaultOids(): Set = buildSet { - connection.createStatement().use { stmt -> - stmt.executeQuery( - "SELECT oid::integer FROM pg_catalog.pg_proc WHERE proname IN ('lag', 'lead') AND pronargs = 3 AND prokind = 'w'", - ).use { rs -> - while (rs.next()) add(rs.getInt("oid")) - } - } - } - /** * Returns NOT NULL column information for views and materialized views in [schemaName], for * [JdbcAnalyzer]'s catalog construction. diff --git a/generator/src/main/kotlin/norm/generator/PgNodeExpression.kt b/generator/src/main/kotlin/norm/generator/PgNodeExpression.kt index 945b4266..f46c3504 100644 --- a/generator/src/main/kotlin/norm/generator/PgNodeExpression.kt +++ b/generator/src/main/kotlin/norm/generator/PgNodeExpression.kt @@ -40,8 +40,8 @@ internal sealed interface PgNodeExpression { * from `:funcvariadic`. In this form the last entry of [arguments] is the array expression * itself, passed through as one value, not exploded into its elements. This matters for * nullability: `concat(VARIADIC arr)` is `null` when `arr` itself is `null` on PostgreSQL - * 16, 17, and 18, which neither [PgCatalogLoader.alwaysNonNullFunctionOids] nor - * [PgCatalogLoader.nonNullIffFirstArgumentNonNullFunctionOids] account for on their own — both + * 16, 17, and 18, which neither [NullabilityCatalog.alwaysNonNullFunctionOids] nor + * [NullabilityCatalog.nonNullIffFirstArgumentNonNullFunctionOids] account for on their own — both * assume the ordinary (non-`VARIADIC`) calling form where every argument is an individual * scalar value, and [NodeTreeNullabilityAnalyzer.isNonNull] must check this flag before * trusting either list unconditionally. @@ -256,11 +256,113 @@ internal sealed interface PgNodeExpression { } } +/** + * Every direct sub-expression of this [PgNodeExpression], in constructor-parameter order. Leaf + * nodes ([PgNodeExpression.Var], [PgNodeExpression.Const], [PgNodeExpression.SqlValueFunction], + * [PgNodeExpression.NextValExpr], [PgNodeExpression.Unknown]) return an empty list. + */ +internal val PgNodeExpression.children: List + get() = when (this) { + is PgNodeExpression.Var, + is PgNodeExpression.Const, + is PgNodeExpression.SqlValueFunction, + is PgNodeExpression.NextValExpr, + is PgNodeExpression.Unknown, + -> emptyList() + + is PgNodeExpression.FuncExpr -> arguments + is PgNodeExpression.OpExpr -> arguments + is PgNodeExpression.ScalarArrayOpExpr -> arguments + is PgNodeExpression.CoalesceExpr -> arguments + is PgNodeExpression.NullIfExpr -> arguments + is PgNodeExpression.MinMaxExpr -> arguments + is PgNodeExpression.Aggref -> arguments + is PgNodeExpression.WindowFunc -> arguments + is PgNodeExpression.SubLink -> listOfNotNull(outerOperand) + is PgNodeExpression.CaseExpr -> + resultExpressions + listOfNotNull(defaultResult, testExpression) + whenConditions + + is PgNodeExpression.BoolExpr -> arguments + is PgNodeExpression.RelabelType -> listOf(argument) + is PgNodeExpression.CoerceViaIo -> listOf(argument) + is PgNodeExpression.ArrayCoerceExpr -> listOf(argument) + is PgNodeExpression.CollateExpr -> listOf(argument) + is PgNodeExpression.CoerceToDomain -> listOf(argument) + is PgNodeExpression.NullTest -> listOf(argument) + is PgNodeExpression.BooleanTest -> listOf(argument) + is PgNodeExpression.DistinctExpr -> arguments + is PgNodeExpression.ArrayExpr -> elements + is PgNodeExpression.RowExpr -> arguments + is PgNodeExpression.GroupingFunc -> arguments + is PgNodeExpression.FieldSelect -> listOf(argument) + is PgNodeExpression.JsonIsPredicate -> listOf(argument) + is PgNodeExpression.JsonConstructorExpr -> arguments + listOfNotNull(function) + is PgNodeExpression.JsonExpr -> listOf(argument) + listOfNotNull(onEmptyDefault, onErrorDefault) + is PgNodeExpression.XmlExpr -> arguments + } + +/** + * Returns a copy of this [PgNodeExpression] with [transform] applied to each of its [children], + * preserving every non-[PgNodeExpression] field as-is. Leaf nodes return themselves unchanged. + */ +internal fun PgNodeExpression.mapChildren(transform: (PgNodeExpression) -> PgNodeExpression): PgNodeExpression = + when (this) { + is PgNodeExpression.Var, + is PgNodeExpression.Const, + is PgNodeExpression.SqlValueFunction, + is PgNodeExpression.NextValExpr, + is PgNodeExpression.Unknown, + -> this + + is PgNodeExpression.FuncExpr -> copy(arguments = arguments.map(transform)) + is PgNodeExpression.OpExpr -> copy(arguments = arguments.map(transform)) + is PgNodeExpression.ScalarArrayOpExpr -> copy(arguments = arguments.map(transform)) + is PgNodeExpression.CoalesceExpr -> copy(arguments = arguments.map(transform)) + is PgNodeExpression.NullIfExpr -> copy(arguments = arguments.map(transform)) + is PgNodeExpression.MinMaxExpr -> copy(arguments = arguments.map(transform)) + is PgNodeExpression.Aggref -> copy(arguments = arguments.map(transform)) + is PgNodeExpression.WindowFunc -> copy(arguments = arguments.map(transform)) + is PgNodeExpression.SubLink -> copy(outerOperand = outerOperand?.let(transform)) + is PgNodeExpression.CaseExpr -> copy( + resultExpressions = resultExpressions.map(transform), + defaultResult = defaultResult?.let(transform), + testExpression = testExpression?.let(transform), + whenConditions = whenConditions.map(transform), + ) + + is PgNodeExpression.BoolExpr -> copy(arguments = arguments.map(transform)) + is PgNodeExpression.RelabelType -> copy(argument = transform(argument)) + is PgNodeExpression.CoerceViaIo -> copy(argument = transform(argument)) + is PgNodeExpression.ArrayCoerceExpr -> copy(argument = transform(argument)) + is PgNodeExpression.CollateExpr -> copy(argument = transform(argument)) + is PgNodeExpression.CoerceToDomain -> copy(argument = transform(argument)) + is PgNodeExpression.NullTest -> copy(argument = transform(argument)) + is PgNodeExpression.BooleanTest -> copy(argument = transform(argument)) + is PgNodeExpression.DistinctExpr -> copy(arguments = arguments.map(transform)) + is PgNodeExpression.ArrayExpr -> copy(elements = elements.map(transform)) + is PgNodeExpression.RowExpr -> copy(arguments = arguments.map(transform)) + is PgNodeExpression.GroupingFunc -> copy(arguments = arguments.map(transform)) + is PgNodeExpression.FieldSelect -> copy(argument = transform(argument)) + is PgNodeExpression.JsonIsPredicate -> copy(argument = transform(argument)) + is PgNodeExpression.JsonConstructorExpr -> copy( + arguments = arguments.map(transform), + function = function?.let(transform), + ) + + is PgNodeExpression.JsonExpr -> copy( + argument = transform(argument), + onEmptyDefault = onEmptyDefault?.let(transform), + onErrorDefault = onErrorDefault?.let(transform), + ) + + is PgNodeExpression.XmlExpr -> copy(arguments = arguments.map(transform)) + } + /** * A CTE (Common Table Expression) definition parsed from a `{COMMONTABLEEXPR ...}` block * in the `:cteList` of a query's node tree. * - * Named with a `NodeTree` prefix to distinguish from [CteDefinition] in `SqlUtils.kt`, which + * Named with a `NodeTree` prefix to distinguish from [CteDefinition] in `SqlCteClause.kt`, which * represents SQL-text-level CTE positions for DML transformation. * * @property name The CTE name (from `:ctename`). @@ -285,15 +387,14 @@ internal data class NodeTreeCteDefinition(val name: String, val queryBlock: Stri * referring back to itself (from `:self_reference`). `false` for every ordinary CTE reference — * this is only ever `true` inside a `WITH RECURSIVE` CTE's own recursive query term. Defaults to * `false` since only [RangeTableEntry.Cte] (built by [PgNodeTreeParser.parseRangeTableEntries]) - * currently reads it; [PgNodeTreeParser.parseCteRangeTableEntries]'s existing callers never did. + * currently reads it; [cteReferences]'s callers never did. */ internal data class NodeTreeCteReference(val name: String, val ctelevelsup: Int, val selfReference: Boolean = false) /** * A single range-table entry, keyed by 1-based `varno`, covering every `rtekind` — unlike - * [PgNodeTreeParser.parseRangeTable] ([Relation] only), [PgNodeTreeParser.parseSubqueryRangeTable] - * ([Subquery] only), and [PgNodeTreeParser.parseCteRangeTableEntries] ([Cte] only), which each - * recognize exactly one kind and silently skip every entry of any other kind. + * [baseRelations] ([Relation] only), [subqueryBlocks] ([Subquery] only), and [cteReferences] ([Cte] + * only), which each derive exactly one kind and silently skip every entry of any other kind. * [NodeTreeProvenanceResolver] walks an arbitrary `Var`'s `varno` and must be able to see an * unrecognized or not-yet-modeled kind ([Other]) so it can bail rather than misinterpret that varno * as one of the recognized kinds. @@ -326,16 +427,65 @@ internal sealed interface RangeTableEntry { */ data class Join(val joinAliasVars: List) : RangeTableEntry + /** + * `rtekind 9`: a PostgreSQL 18+ `*GROUP*` RTE whose `:groupexprs` was present and parsed — see + * [groupExpressions] and [groupRteMap], the two derived views built from this entry. + * + * @property groupExpressionBlocks The raw `{...}` blocks of `:groupexprs`, in declaration order — + * a target-list `Var` referencing this RTE's `varno` resolves to + * `groupExpressionBlocks[varattno - 1]` (1-based `varattno` indexing into the 0-based list). + */ + data class Group(val groupExpressionBlocks: List) : RangeTableEntry + /** * Any `rtekind` this parser does not model individually: `3` (function), `4` (tablefunc, e.g. - * `JSON_TABLE`), `5` (`VALUES`), `7` (named tuplestore), `8` (result, a FROM-less `SELECT`), or - * `9` (a PostgreSQL 18+ `*GROUP*` RTE — see [PgNodeTreeParser.parseGroupRteMap]). Named - * `rtekind`, not e.g. `kind`, to match the field name so a reader cross-referencing raw - * `pg_node_tree` text does not need to translate. + * `JSON_TABLE`), `5` (`VALUES`), `7` (named tuplestore), `8` (result, a FROM-less `SELECT`), or a + * `9` (`*GROUP*` RTE) whose `:groupexprs` could not be parsed — see [Group] for the ordinary + * rtekind 9 case. Named `rtekind`, not e.g. `kind`, to match the field name so a reader + * cross-referencing raw `pg_node_tree` text does not need to translate. */ data class Other(val rtekind: Int) : RangeTableEntry } +/** Varno to `relid`, for every `rtekind 0` (base table) entry. */ +internal fun Map.baseRelations(): Map = + mapNotNull { (varno, entry) -> (entry as? RangeTableEntry.Relation)?.let { varno to it.relid } }.toMap() + +/** Varno to `:subquery` block text, for every `rtekind 1` (derived table) entry. */ +internal fun Map.subqueryBlocks(): Map = + mapNotNull { (varno, entry) -> (entry as? RangeTableEntry.Subquery)?.let { varno to it.queryBlock } }.toMap() + +/** Varno to CTE reference, for every `rtekind 6` entry. */ +internal fun Map.cteReferences(): Map = + mapNotNull { (varno, entry) -> (entry as? RangeTableEntry.Cte)?.let { varno to it.reference } }.toMap() + +/** + * The GROUP RTE's `varno` to its `:groupexprs` list, fully parsed — see [RangeTableEntry.Group]. + * Unlike [groupRteMap], a grouping key that is not a bare `Var` (e.g. `lower(a)`, a literal, or a + * `Var` carrying outer-join `:varnullingrels`) is preserved here rather than reduced to nothing. + */ +internal fun Map.groupExpressions(parser: PgNodeTreeParser): Map> = + mapNotNull { (varno, entry) -> + (entry as? RangeTableEntry.Group)?.let { varno to it.groupExpressionBlocks.map(parser::parseExpression) } + }.toMap() + +/** + * `(groupVarno, 1-based attribute position)` to `(baseVarno, baseVarattno)`, read textually via + * [PgNodeTreeParser.firstVarnoAndVarattno] — the first `:varno` and `:varattno` in each + * `:groupexprs` block, never [PgNodeTreeParser.parseExpression]. A block missing either field is + * skipped without shifting the 1-based attribute position of the blocks after it. + */ +internal fun Map.groupRteMap(parser: PgNodeTreeParser): Map, Pair> = + buildMap { + for ((groupVarno, entry) in this@groupRteMap) { + if (entry !is RangeTableEntry.Group) continue + entry.groupExpressionBlocks.forEachIndexed { attrIndex, block -> + val (baseVarno, baseVarattno) = parser.firstVarnoAndVarattno(block) ?: return@forEachIndexed + put(groupVarno to (attrIndex + 1), baseVarno to baseVarattno) + } + } + } + /** * A single result column from a query's `targetList`. * diff --git a/generator/src/main/kotlin/norm/generator/PgNodeTreeParser.kt b/generator/src/main/kotlin/norm/generator/PgNodeTreeParser.kt index 41f661aa..e3a5319b 100644 --- a/generator/src/main/kotlin/norm/generator/PgNodeTreeParser.kt +++ b/generator/src/main/kotlin/norm/generator/PgNodeTreeParser.kt @@ -91,107 +91,6 @@ internal class PgNodeTreeParser { PgNodeExpression.Unknown("PARSE_ERROR") } - /** - * Parses the range table from a full `pg_node_tree` text into a map from 1-based `varno` to `relid` OID. - * - * Extracts the `:rtable` section from [nodeTreeText] at the outermost QUERY level, splits it - * into `{RANGETBLENTRY ...}` blocks, and returns a map from 1-based position index (varno) to - * the `:relid` OID for each entry with `rtekind 0` (regular base table). - * - * Entries with `rtekind != 0` (subqueries with `rtekind 1`, joins with `rtekind 2`, - * functions with `rtekind 3`, CTEs with `rtekind 6`, etc.) are skipped and contribute `null` - * for their position — their varnos do not appear as keys in the returned map. - * - * @param nodeTreeText the raw `pg_rewrite.ev_action` text - * @return a map from 1-based varno to `relid` OID for base table range table entries only, - * or an empty map if [nodeTreeText] is malformed or contains no `:rtable` - */ - fun parseRangeTable(nodeTreeText: String): Map { - val rtableContent = extractOuterSectionContent(nodeTreeText, ":rtable (") ?: return emptyMap() - return buildMap { - splitBraceBlocks(rtableContent).forEachIndexed { index, rangeTableEntry -> - val rtekind = extractIntField(rangeTableEntry, ":rtekind") ?: return@forEachIndexed - if (rtekind != 0) return@forEachIndexed - val relid = extractIntField(rangeTableEntry, ":relid") ?: return@forEachIndexed - put(index + 1, relid) // varno is 1-based - } - } - } - - /** - * Parses GROUP BY RTE entries (rtekind 9) from a full `pg_node_tree` text. - * - * PostgreSQL creates an `*GROUP*` range table entry (rtekind 9) for aggregate queries with - * GROUP BY. Target list VARs for grouped columns reference this GROUP RTE (using its varno) - * rather than the base table directly. Each GROUP RTE holds a `:groupexprs` list of VARs - * pointing back to the original base table columns. - * - * Example: `SELECT author.name, COUNT(*) FROM author JOIN book ... GROUP BY author.name` - * produces a target list with `Var(varno=4, varattno=1)` where varno=4 is the GROUP RTE and - * varattno=1 selects the first entry in `:groupexprs` — `Var(varno=1, varattno=2)` (author.name). - * - * @return a map from `(groupVarno, 1-based attribute position)` to `(baseVarno, baseVarattno)`, - * enabling resolution of GROUP BY column references back to their source base table columns; - * or an empty map if the node tree contains no GROUP BY RTEs or is malformed - */ - fun parseGroupRteMap(nodeTreeText: String): Map, Pair> { - val rtableContent = extractOuterSectionContent(nodeTreeText, ":rtable (") ?: return emptyMap() - return buildMap { - splitBraceBlocks(rtableContent).forEachIndexed { index, rangeTableEntry -> - val rtekind = extractIntField(rangeTableEntry, ":rtekind") ?: return@forEachIndexed - if (rtekind != 9) return@forEachIndexed - val groupVarno = index + 1 - // extractOuterSectionContent works on any {NODE ...} block at its "outer" level (depth 1). - val groupExprsContent = extractOuterSectionContent(rangeTableEntry, ":groupexprs (") - ?: return@forEachIndexed - splitBraceBlocks(groupExprsContent).forEachIndexed { attrIndex, varBlock -> - val baseVarno = extractIntField(varBlock, ":varno") ?: return@forEachIndexed - val baseVarattno = extractIntField(varBlock, ":varattno") ?: return@forEachIndexed - put(groupVarno to (attrIndex + 1), baseVarno to baseVarattno) - } - } - } - } - - /** - * Parses GROUP RTE (`rtekind 9`) group expressions — the fully-parsed counterpart to - * [parseGroupRteMap] — from a full `pg_node_tree` text. - * - * PostgreSQL 18 introduced an `RTE_GROUP` range-table entry (`:rtekind 9`, alias `*GROUP*`) that - * carries a `:groupexprs` list of the query's grouping-key expressions, and rewrites every - * target-list occurrence of a grouping-key expression (not just the one PostgreSQL assigns - * `:ressortgroupref` to) into a bare `Var` referencing this RTE. PostgreSQL 16 and 17 have no - * such RTE — no `:rtable` entry there ever has `:rtekind 9` — so on those versions this method - * always returns an empty map, and the original expression is left in place in the target list - * for [parseTargetList] to see directly. - * - * Unlike [parseGroupRteMap], which only resolves a GROUP RTE entry that is itself a bare `VAR` - * (mapping it back to a `(baseVarno, baseVarattno)` pair), this method parses the `:groupexprs` - * entry into a full [PgNodeExpression] — a grouping key can be an arbitrary expression (e.g. - * `lower(a)`, a literal, or a `Var` carrying outer-join `:varnullingrels`), not only a bare - * column reference. See [substituteGroupRteVars] for how a target-list `Var` referencing this - * RTE is substituted back to the expression this method returns. - * - * @return a map from the GROUP RTE's 1-based `varno` (its position in `:rtable`) to its parsed - * `:groupexprs` list, in declaration order — a target-list `Var` whose `:varno` is a key here - * resolves to `list[varattno - 1]` (1-based `:varattno` indexing into the 0-based list). Empty - * when [nodeTreeText] is malformed, absent, or contains no GROUP RTE (including every - * PostgreSQL 16/17 tree, and any PostgreSQL 18+ tree for a query with no `GROUP BY`). - */ - fun parseGroupRteExpressions(nodeTreeText: String): Map> { - val rtableContent = extractOuterSectionContent(nodeTreeText, ":rtable (") ?: return emptyMap() - return buildMap { - splitBraceBlocks(rtableContent).forEachIndexed { index, rangeTableEntry -> - val rtekind = extractIntField(rangeTableEntry, ":rtekind") ?: return@forEachIndexed - if (rtekind != 9) return@forEachIndexed - val groupVarno = index + 1 - val groupExprsContent = extractOuterSectionContent(rangeTableEntry, ":groupexprs (") - ?: return@forEachIndexed - put(groupVarno, splitBraceBlocks(groupExprsContent).map(::parseExpression)) - } - } - } - /** * Returns `true` if the outermost QUERY node in [nodeTreeText] has a `:setOperations` field. * @@ -374,31 +273,6 @@ internal class PgNodeTreeParser { return (fromGroupClause + fromGroupingSets).toSet() } - /** - * Parses subquery range table entries from a full `pg_node_tree` text. - * - * Extracts the `:rtable` section from [nodeTreeText] at the outermost QUERY level, and for each - * entry with `rtekind 1` (subquery), extracts the embedded `:subquery {QUERY ...}` block. - * - * @param nodeTreeText the raw `pg_rewrite.ev_action` text (or a bare `{QUERY ...}` block) - * @return a map from 1-based varno to the subquery's `{QUERY ...}` block text, or empty if none - */ - fun parseSubqueryRangeTable(nodeTreeText: String): Map { - val rtableContent = extractOuterSectionContent(nodeTreeText, ":rtable (") ?: return emptyMap() - return buildMap { - splitBraceBlocks(rtableContent).forEachIndexed { index, rangeTableEntry -> - val rtekind = extractIntField(rangeTableEntry, ":rtekind") ?: return@forEachIndexed - if (rtekind != 1) return@forEachIndexed - val subqueryMarker = ":subquery {" - val subqueryIndex = rangeTableEntry.indexOf(subqueryMarker) - if (subqueryIndex == -1) return@forEachIndexed - val braceStart = subqueryIndex + subqueryMarker.length - 1 - val subqueryBlock = extractBalancedBraces(rangeTableEntry, braceStart) ?: return@forEachIndexed - put(index + 1, subqueryBlock) // varno is 1-based - } - } - } - /** * Parses the CTE definitions from a full `pg_node_tree` text. * @@ -433,40 +307,11 @@ internal class PgNodeTreeParser { } } - /** - * Parses CTE range table entries (`rtekind 6`) from a full `pg_node_tree` text. - * - * Extracts the `:rtable` section and returns a map from 1-based `varno` to a - * [NodeTreeCteReference] (the CTE's `:ctename` and `:ctelevelsup`) for each range table entry - * with `rtekind 6`. `:ctelevelsup` defaults to `0` when absent, matching PostgreSQL's own default - * for a same-level reference; on PostgreSQL 18 the field is always present on a real CTE RTE, so - * this default is defensive only. - * - * This is the CTE counterpart to [parseRangeTable] (which handles `rtekind 0` base tables) - * and [parseSubqueryRangeTable] (which handles `rtekind 1` subqueries). - * - * @param nodeTreeText the raw `pg_rewrite.ev_action` text (or a bare `{QUERY ...}` block) - * @return a map from 1-based varno to [NodeTreeCteReference], or an empty map if no CTE RTEs are - * found - */ - fun parseCteRangeTableEntries(nodeTreeText: String): Map { - val rtableContent = extractOuterSectionContent(nodeTreeText, ":rtable (") ?: return emptyMap() - return buildMap { - splitBraceBlocks(rtableContent).forEachIndexed { index, rangeTableEntry -> - val rtekind = extractIntField(rangeTableEntry, ":rtekind") ?: return@forEachIndexed - if (rtekind != 6) return@forEachIndexed - val cteName = extractStringField(rangeTableEntry, ":ctename") ?: return@forEachIndexed - val ctelevelsup = extractIntField(rangeTableEntry, ":ctelevelsup") ?: 0 - put(index + 1, NodeTreeCteReference(name = cteName, ctelevelsup = ctelevelsup)) - } - } - } - /** * Parses every range-table entry from [nodeTreeText]'s own `:rtable`, regardless of `rtekind`, * into a [RangeTableEntry] — see that type's KDoc for why a resolver needs visibility into every - * kind, not just the ones [parseRangeTable], [parseSubqueryRangeTable], and - * [parseCteRangeTableEntries] each recognize individually. + * kind, not just the ones [baseRelations], [subqueryBlocks], and [cteReferences] each derive + * individually. * * None of the fields read here need [findMarkerAtDepthOne]'s depth-one-awareness: a `JOINEXPR` * range-table entry's own fields (`:jointype`, `:joinaliasvars`, etc.) contain no nested `QUERY` @@ -503,6 +348,15 @@ internal class PgNodeTreeParser { val selfReference = extractBoolField(rangeTableEntry, ":self_reference") ?: false RangeTableEntry.Cte(NodeTreeCteReference(cteName, ctelevelsup, selfReference)) } + 9 -> { + // extractOuterSectionContent works on any {NODE ...} block at its "outer" level (depth 1). + val groupExprsContent = extractOuterSectionContent(rangeTableEntry, ":groupexprs (") + if (groupExprsContent != null) { + RangeTableEntry.Group(splitBraceBlocks(groupExprsContent)) + } else { + RangeTableEntry.Other(rtekind) + } + } else -> RangeTableEntry.Other(rtekind) } put(index + 1, entry) // varno is 1-based @@ -586,7 +440,7 @@ internal class PgNodeTreeParser { * A `MERGE` with no `DELETE` action anywhere — only `UPDATE`/`INSERT` actions — always leaves a * written or freshly-inserted row behind for `RETURNING` to see, so its `NEW` reference is exactly * as trustworthy as an ordinary column; only the presence of a `DELETE` action makes `NEW` - * unconditionally forced nullable (see [PgCatalogLoader.forcesNewNullable]'s caller). + * unconditionally forced nullable (see [ColumnNullabilityAnalyzer.forcesNewNullable]'s caller). * * @return `false` for a non-`MERGE` statement (`:mergeActionList` is absent), or for a `MERGE` * with no `DELETE` action @@ -702,11 +556,11 @@ internal class PgNodeTreeParser { } val testExpressionOperatorOid = (testExprBlock?.let(::parseExpression) as? PgNodeExpression.OpExpr) ?.operatorFunctionOid - // :subselect holds the sublink's subquery body ({QUERY ...}), mirroring parseSubqueryRangeTable - // and parseCteList's extraction of the same node shape. Depth-one-awareness (via - // extractFieldExpression) is required: :testexpr precedes :subselect in SUBLINK's field order, - // and :testexpr's own value can contain a nested sublink with its own :subselect — see - // extractFieldExpression's KDoc for the repro this guards against. + // :subselect holds the sublink's subquery body ({QUERY ...}), mirroring parseRangeTableEntries's + // own :subquery extraction and parseCteList's extraction of the same node shape. + // Depth-one-awareness (via extractFieldExpression) is required: :testexpr precedes :subselect in + // SUBLINK's field order, and :testexpr's own value can contain a nested sublink with its own + // :subselect — see extractFieldExpression's KDoc for the repro this guards against. val subselectBlock = extractFieldExpression(text, ":subselect") return PgNodeExpression.SubLink( subLinkType = subLinkType, @@ -1012,6 +866,19 @@ internal class PgNodeTreeParser { intFieldPatterns.getOrPut(fieldName) { Regex("""$fieldName (-?\d+)""") } .find(text)?.groupValues?.get(1)?.toIntOrNull() + /** + * The first textual `:varno` and `:varattno` in [block], via [extractIntField] — not + * [parseExpression] — so a `:groupexprs` entry that is not itself a bare `VAR` (e.g. a + * `FUNCEXPR` wrapping one) still yields the `VAR` nested inside it. Used by `groupRteMap`. + * + * @return `null` if either field is absent. + */ + internal fun firstVarnoAndVarattno(block: String): Pair? { + val varno = extractIntField(block, ":varno") ?: return null + val varattno = extractIntField(block, ":varattno") ?: return null + return varno to varattno + } + /** * Extracts a boolean field value from a node block. * diff --git a/generator/src/main/kotlin/norm/generator/PostgresBaseTypes.kt b/generator/src/main/kotlin/norm/generator/PostgresBaseTypes.kt new file mode 100644 index 00000000..289d7754 --- /dev/null +++ b/generator/src/main/kotlin/norm/generator/PostgresBaseTypes.kt @@ -0,0 +1,236 @@ +// PostgresBaseType is the only top-level class in this file, but the file also holds +// POSTGRES_BASE_TYPES and the functions that read it (resolveWireCodec, +// postgresArrayElementTypeName) — deliberately, per this file's role as the single home for +// Postgres base-type mapping, not a naming slip. +@file:Suppress("MatchingDeclarationName") + +package norm.generator + +import com.squareup.kotlinpoet.asClassName +import com.squareup.kotlinpoet.asTypeName +import java.math.BigDecimal +import java.sql.Blob +import java.time.LocalDate +import java.time.LocalDateTime +import java.time.LocalTime +import java.time.OffsetTime +import java.util.UUID + +/** + * One Postgres base type Norm maps to Kotlin, keyed in [POSTGRES_BASE_TYPES] by its canonical or + * SQL-standard spelling. + * + * @property codec Wire-level JDBC access for this type, shared by a plain (adapterless) column's + * [ScalarSqlMappable] and by the same type behind a `ColumnAdapter` (an enum, a domain, or a + * user type mapping). + */ +internal class PostgresBaseType(val codec: WireCodec) + +/** + * Every canonical Postgres base type name [TypeRepository.resolveBaseType] accepts, keyed by name + * (after stripping a `pg_catalog.` qualification — see [TypeRepository.resolveBaseType]). + * + * Includes the `serial`/`smallserial`/`bigserial` pseudo-types even though Postgres rejects + * `CREATE DOMAIN ... AS serial` outright (`type "serial" does not exist` on a live server; a + * domain's base is always a real, registered `pg_type`, so `domain.baseType` can never actually be + * one of these), and even though [resolveWireCodec]'s only other callers + * ([TypeRepository.buildUserConfiguredMappable]'s user-configured type mappings) also only ever see + * the real, JDBC-reported type name, never a serial alias: a plain column still needs + * [PostgresBaseType.codec] for one, so the row belongs here regardless. + * + * [register]'s `check` guards against a name repeated across two calls silently overwriting the + * earlier row — `vararg` alone would let that pass unnoticed. + */ +internal val POSTGRES_BASE_TYPES: Map = buildMap { + fun register(codec: WireCodec, vararg names: String) { + for (name in names) { + check(put(name, PostgresBaseType(codec)) == null) { "duplicate base type name: $name" } + } + } + + register( + PrimitiveCodec(Short::class.asTypeName(), "Short", "SMALLINT"), + "smallserial", + "serial2", + "smallint", + "int2", + ) + + register( + PrimitiveCodec(Int::class.asTypeName(), "Int", "INTEGER"), + "serial", + "serial4", + "integer", + "int", + "int4", + ) + + register( + PrimitiveCodec(Long::class.asTypeName(), "Long", "BIGINT"), + "bigserial", + "serial8", + "bigint", + "int8", + ) + + register( + PrimitiveCodec(Float::class.asTypeName(), "Float", "REAL"), + "real", + "float4", + ) + + register( + PrimitiveCodec(Double::class.asTypeName(), "Double", "DOUBLE"), + "float", + "double precision", + "float8", + ) + + register( + PrimitiveCodec(Boolean::class.asTypeName(), "Boolean", "BOOLEAN"), + "bool", + "boolean", + ) + + register( + ObjectGetterCodec(BigDecimal::class.asTypeName(), "getBigDecimal", "setBigDecimal", "NUMERIC"), + "numeric", + ) + + // json and jsonb require setObject(..., Types.OTHER): Postgres JDBC rejects setString() for both + // in prepared statements, just as it does for enum columns. TypeRepository's ENUM_CODEC reuses + // this exact codec, so the binding for a plain json/jsonb column and for an enum column can never + // drift apart. + register( + TypesOtherCodec(String::class.asTypeName(), "getString", "OTHER"), + "json", + "jsonb", + ) + + register( + ObjectGetterCodec(String::class.asTypeName(), "getString", "setString", "VARCHAR"), + "text", + "varchar", + "bpchar", + "string", + ) + + // Scalar oid maps to Blob: pgjdbc's setBlob() creates a Postgres large object and stores its oid, + // the standard large-object convention, via the plain named getBlob()/setBlob() methods (no + // class-hint or Types constant needed). oid[] does not share this mapping (see + // TypeRepository.tryResolveStandardType) because an array of large-object handles has no coherent + // JDBC semantics, and real-world oid[] columns hold plain catalog identifiers, not large objects. + register( + ObjectGetterCodec(Blob::class.asTypeName(), "getBlob", "setBlob", "BLOB"), + "oid", + ) + + // java.sql.ResultSet.getBytes/PreparedStatement.setBytes are plain named methods for bytea, + // needing no class-hint. + register( + ObjectGetterCodec(ByteArray::class.asTypeName(), "getBytes", "setBytes", "BINARY"), + "bytea", + ) + + // pgjdbc's plain getObject(int) returns java.sql.Date/Time/Timestamp for date/time/timetz/ + // timestamp columns, not the java.time type, so the read needs the class-qualified + // getObject(int, Class) overload (ClassHintedObjectCodec). The write side needs no such + // qualification: PgPreparedStatement.setObject(int, Object) already dispatches on the runtime + // type of a LocalDate/LocalTime/OffsetTime/LocalDateTime/OffsetDateTime argument directly + // (pgjdbc 42.7.13's source). + register( + ClassHintedObjectCodec(LocalDate::class.asClassName(), "DATE"), + "date", + ) + + register( + ClassHintedObjectCodec(LocalTime::class.asClassName(), "TIME"), + "time", + ) + + register( + ClassHintedObjectCodec(OffsetTime::class.asClassName(), "TIME_WITH_TIMEZONE"), + "timetz", + ) + + register( + ClassHintedObjectCodec(LocalDateTime::class.asClassName(), "TIMESTAMP"), + "timestamp", + ) + + // InstantViaOffsetDateTimeCodec's wire representation is OffsetDateTime (read via the + // class-qualified getObject, written via plain setObject — both checked against pgjdbc's source + // the same way as the other java.time entries above), but the Kotlin representation is Instant, + // via a `.toInstant()`/`OffsetDateTime.ofInstant(...)` conversion — see + // InstantViaOffsetDateTimeCodec's KDoc. + register( + InstantViaOffsetDateTimeCodec, + "timestamptz", + ) + + // java.sql.ResultSet.getObject(int) is declared to return Object, so a bare getObject(index) call + // is statically Any in Kotlin regardless of what pgjdbc returns at runtime — PgResultSet's + // internalGetObject does special-case the Postgres "uuid" type by name and hands back a + // java.util.UUID instance (per pgjdbc 42.7.13's source), but that's a runtime fact, not a static + // type, and a `ColumnAdapter.decode` call requires a statically-typed UUID + // argument. The class-qualified getObject(int, Class) overload (ClassHintedObjectCodec) fixes the + // static type; pgjdbc's PgResultSet#getObject(int, Class) explicitly special-cases + // `type == UUID.class` by delegating to the same runtime read and casting, so this is safe. + register( + ClassHintedObjectCodec(UUID::class.asClassName(), "OTHER"), + "uuid", + ) +} + +/** + * Canonicalizes a Postgres type name for use as the element type of + * [java.sql.Connection.createArrayOf]. + * + * The driver appends `[]` to this name and looks the result up in `pg_type`, so it must be a + * canonical `pg_type` name. [TypeRepository.resolveBaseType] additionally accepts SQL spellings + * (`integer`, `boolean`, `double precision`) and `pg_catalog.`-qualified names; without folding + * those here, `postgresArrayElementTypeName("integer")` would return `"integer"` verbatim and + * `createArrayOf` would fail with `Unable to find server array type for provided name {0}`, since + * `pg_type` has no row named `integer` — only `int4`. + * + * Every branch below was checked against a live PostgreSQL 17 server via + * `SELECT typname FROM pg_type WHERE oid = to_regtype(?)`: every alias here resolves to the + * canonical name on its right-hand side, and every `pg_catalog.`-qualified spelling of an + * already-canonical name (e.g. `pg_catalog.uuid`, `pg_catalog.timestamptz`) resolves to itself — + * confirming the universal `removePrefix` below is sufficient for those without a dedicated + * branch. `pg_catalog.boolean` and `pg_catalog.integer` do not resolve on a live server (`boolean` + * and `integer` are SQL-standard keyword aliases recognized only unqualified, not as schema- + * qualified `pg_catalog` names) — but that combination can never actually reach this function: + * JDBC's `TYPE_NAME`/`getColumnTypeName` always report the canonical, unqualified name. + * + * `serial` and its variants need no entry: Postgres has no serial array type, so a serial column + * can never reach the array path. + */ +internal fun postgresArrayElementTypeName(typeName: String): String = + when (val canonical = typeName.removePrefix("pg_catalog.")) { + "smallint" -> "int2" + "integer", "int" -> "int4" + "bigint" -> "int8" + "real" -> "float4" + "double precision", "float" -> "float8" + "boolean" -> "bool" + "string" -> "text" + else -> canonical + } + +/** + * Maps a Postgres base type name to its [WireCodec], or returns `null` if unsupported. + * + * Delegates to [POSTGRES_BASE_TYPES], the single source of truth for both a plain column's + * [SqlMappable] and its wire-level [WireCodec] — a domain over any base type + * [TypeRepository.resolveBaseType] itself supports (e.g. `CREATE DOMAIN d AS timestamptz`) always + * resolves here too, since both come from the same row. [TypeRepository]'s domain resolution + * chains through this function (see [TypeRepository.tryResolveDomainType] and + * [domainKotlinBaseType][norm.generator.domainKotlinBaseType]); its `error()` calls are reachable + * only for a base type [TypeRepository.resolveBaseType] itself does not support either (e.g. `xml`, + * `interval`, `money` — Postgres allows a domain over any of these, but Norm has never mapped them + * to a Kotlin type as a plain column type, so the same limitation applies to a domain built on + * one). That failure is intentional: a clear, immediate `error()` naming the unsupported type is + * preferable to silently guessing a mapping for a type Norm has no tested behavior for. + */ +internal fun resolveWireCodec(baseTypeName: String): WireCodec? = POSTGRES_BASE_TYPES[baseTypeName]?.codec diff --git a/generator/src/main/kotlin/norm/generator/QueryFileParser.kt b/generator/src/main/kotlin/norm/generator/QueryFileParser.kt index 48c6702d..4d5577d4 100644 --- a/generator/src/main/kotlin/norm/generator/QueryFileParser.kt +++ b/generator/src/main/kotlin/norm/generator/QueryFileParser.kt @@ -52,7 +52,8 @@ public data class ParsedQuery( * a named parameter produces its own `?` — the same name used multiple times creates multiple * bind slots. Mixing `:name` and `?` styles in a single query is not allowed. * - * Named parameters inside single-quoted string literals are left untouched. + * A `:name` that appears inside a string literal, quoted identifier, dollar-quoted string, or + * comment is left untouched. */ public object QueryFileParser { @@ -160,14 +161,21 @@ public object QueryFileParser { /** * Converts `:paramName` named parameters to `?` positional placeholders. * - * Scans the SQL character by character to correctly handle: - * - `::` cast operators (e.g., `value::integer`) — skipped - * - Single-quoted string literals (e.g., `':notaparam'`) — skipped, including `''` escapes - * - SQL comments (e.g., `-- comment`) — skipped + * Scans the SQL one lexical token at a time via [skipLexicalToken], so a `:name`-shaped run + * inside a string literal, quoted identifier, dollar-quoted string, or comment is copied through + * verbatim rather than converted. Outside those tokens: + * - `::` cast operators (e.g., `value::integer`) are passed through unconverted. + * - A `:` immediately followed by an ASCII `[A-Za-z_]` character (see [isNamedParameterStartCharacter]) + * is a named parameter and becomes `?`; the name itself continues through + * [isNamedParameterCharacter]. * * Each occurrence of a named parameter produces its own `?` placeholder with its own 1-based * position number. The same name appearing multiple times creates multiple bind slots. * + * Every `?` encountered outside a lexical token — meaning it was written directly into the SQL, + * not produced by this conversion — is counted so the mixed-style guard below can tell a + * genuine positional `?` apart from one merely sitting inside a string literal like `'really?'`. + * * @return A pair of (converted SQL, position-to-name map). If the SQL has no named parameters, * returns the original SQL with an empty map. * @throws IllegalArgumentException if the query mixes `:name` and `?` parameter styles. @@ -175,46 +183,38 @@ public object QueryFileParser { private fun convertNamedParameters(sql: String): Pair> { val numberToName = mutableMapOf() var nextNumber = 1 + var positionalParameterCount = 0 val result = StringBuilder() - var i = 0 - - while (i < sql.length) { - val c = sql[i] - - if (c == '\'') { - // Skip single-quoted string literals - val closeIndex = findClosingQuote(sql, i) - result.append(sql, i, closeIndex + 1) - i = closeIndex + 1 - } else if (c == '-' && i + 1 < sql.length && sql[i + 1] == '-') { - // Skip -- line comments - val eol = sql.indexOf('\n', i) - if (eol < 0) { - result.append(sql, i, sql.length) - i = sql.length - } else { - result.append(sql, i, eol) - i = eol - } - } else if (c == ':' && i + 1 < sql.length && sql[i + 1] == ':') { + var index = 0 + + while (index < sql.length) { + val afterToken = skipLexicalToken(sql, index) + if (afterToken != index) { + result.append(sql, index, afterToken) + index = afterToken + continue + } + val character = sql[index] + if (character == ':' && index + 1 < sql.length && sql[index + 1] == ':') { // Double colon (cast) — pass through both characters result.append("::") - i += 2 - } else if (c == ':' && i + 1 < sql.length && isIdentifierStart(sql[i + 1])) { + index += 2 + } else if (character == ':' && index + 1 < sql.length && isNamedParameterStartCharacter(sql[index + 1])) { // Named parameter — each occurrence gets its own ? placeholder - val nameStart = i + 1 + val nameStart = index + 1 var nameEnd = nameStart - while (nameEnd < sql.length && isIdentifierPart(sql[nameEnd])) { + while (nameEnd < sql.length && isNamedParameterCharacter(sql[nameEnd])) { nameEnd++ } val paramName = sql.substring(nameStart, nameEnd) val position = nextNumber++ numberToName[position] = paramName result.append('?') - i = nameEnd + index = nameEnd } else { - result.append(c) - i++ + if (character == '?') positionalParameterCount++ + result.append(character) + index++ } } @@ -222,38 +222,21 @@ public object QueryFileParser { return sql to emptyMap() } - // Check for mixed styles: named params found, but ? positional params also present in the original SQL - require('?' !in sql) { + // Check for mixed styles: named params found, but a genuine positional ? also appears outside + // any lexical token — a ? inside a string literal like 'really?' does not count (see + // positionalParameterCount's accumulation above). + require(positionalParameterCount == 0) { "Cannot mix named (:param) and positional (?) parameters in the same query" } return result.toString() to numberToName } - /** - * Finds the closing single quote for a string literal starting at [start]. - * Handles `''` escape sequences (two consecutive single quotes inside a literal). - */ - private fun findClosingQuote(sql: String, start: Int): Int { - var i = start + 1 - while (i < sql.length) { - if (sql[i] == '\'') { - // Check for '' escape - if (i + 1 < sql.length && sql[i + 1] == '\'') { - i += 2 - continue - } - return i - } - i++ - } - // Unterminated string — return end of string - return sql.length - 1 - } - - private fun isIdentifierStart(c: Char): Boolean = c in 'a'..'z' || c in 'A'..'Z' || c == '_' + private fun isNamedParameterStartCharacter(character: Char): Boolean = + character in 'a'..'z' || character in 'A'..'Z' || character == '_' - private fun isIdentifierPart(c: Char): Boolean = isIdentifierStart(c) || c in '0'..'9' + private fun isNamedParameterCharacter(character: Char): Boolean = + isNamedParameterStartCharacter(character) || character in '0'..'9' private fun extractCommentText(commentLine: String): String = commentLine.removePrefix("--").trim() } diff --git a/generator/src/main/kotlin/norm/generator/SqlCteClause.kt b/generator/src/main/kotlin/norm/generator/SqlCteClause.kt index 9e288f73..9b7258ee 100644 --- a/generator/src/main/kotlin/norm/generator/SqlCteClause.kt +++ b/generator/src/main/kotlin/norm/generator/SqlCteClause.kt @@ -16,20 +16,12 @@ package norm.generator * double quotes if the user quoted it. Safe to splice verbatim into a `FROM ` probe. * @property bodyOpenParenthesis Index of `(` that opens the CTE body in the original SQL. * @property bodyCloseParenthesis Index of `)` that closes the CTE body. - * @property hasColumnList Whether the CTE was declared with an explicit column list - * (`name(col1, col2) AS (...)`). The list renames/repositions the body's own output names — but - * [resolveNodeTreeProvenanceExpression] never consults this flag: it cross-validates against the - * CTE body's own `:resname`s (read from the node tree, via [PgNodeTreeParser.parseTargetList]), - * which an explicit column list never changes, so a renamed CTE still resolves correctly. Kept - * for callers that need to know a column list was present, not because expression resolution - * depends on it. */ internal data class CteDefinition( val name: String, val rawName: String, val bodyOpenParenthesis: Int, val bodyCloseParenthesis: Int, - val hasColumnList: Boolean = false, ) /** @@ -37,17 +29,8 @@ internal data class CteDefinition( * * @property definitions The CTEs in declaration order. * @property mainQueryStart Index in the original SQL where the main query (after all CTEs) begins. - * @property isRecursive Whether the clause is `WITH RECURSIVE`. This determines name visibility: - * under `WITH RECURSIVE`, every CTE's body can see every other CTE in the clause (including - * ones declared later); under a plain `WITH`, a CTE's body can only see CTEs declared before - * it (and, per SQL semantics, an outer name with the same name as a later CTE — a later CTE - * never shadows for an earlier body). */ -internal data class ParsedCteClause( - val definitions: List, - val mainQueryStart: Int, - val isRecursive: Boolean, -) +internal data class ParsedCteClause(val definitions: List, val mainQueryStart: Int) /** * Parses CTE definitions from a SQL `WITH` clause. @@ -75,9 +58,7 @@ internal fun parseCteClause(sql: String): ParsedCteClause? { position = skipWhitespaceAndComments(sql, position) // Skip optional RECURSIVE keyword (word-boundary check avoids matching CTE names like "recursive_cte") - val positionAfterRecursive = skipOptionalKeyword(sql, position, "RECURSIVE") - val isRecursive = positionAfterRecursive != position - position = positionAfterRecursive + position = skipOptionalKeyword(sql, position, "RECURSIVE") val definitions = mutableListOf() @@ -92,7 +73,7 @@ internal fun parseCteClause(sql: String): ParsedCteClause? { return if (definitions.isEmpty()) { null } else { - ParsedCteClause(definitions, position, isRecursive) + ParsedCteClause(definitions, position) } } @@ -129,9 +110,7 @@ private fun parseSingleCteDefinition(sql: String, startPosition: Int): Pair= 0x80` (see [matchTrailingAliasSegment], which enforces the same - * rule for an implicit alias's own first character) — never a digit or `$`, both legal only after - * the first character (see [isIdentifierChar]). + * The character class an unquoted PostgreSQL identifier's first character may be: a letter, `_`, + * or any character whose code is `>= 0x80` — never a digit or `$`, which are legal only after the + * first character. */ internal const val COLUMN_REFERENCE_IDENTIFIER_START = """[\p{L}_\x{80}-\x{10FFFF}]""" /** - * The character class an unquoted PostgreSQL identifier's characters after the first may be — - * the same class [isIdentifierChar] checks, expressed as a regex character class: a Unicode - * letter (`\p{L}`, matching [Char.isLetter]) or decimal digit (`\p{Nd}`, matching [Char.isDigit]), - * `_`, `$`, or any character whose code is `>= 0x80` (`\x{80}-\x{10FFFF}`, a code-point range, not - * a per-`Char` one — this is what lets it match a supplementary-plane character written as a - * surrogate pair in a Kotlin `String`: `Regex("[\\x{80}-\\x{10FFFF}]").matches` on a single - * surrogate-pair string returns `true`, consuming both UTF-16 code units as the one code point - * they represent, rather than needing the range repeated for each half). + * The character class an unquoted PostgreSQL identifier's characters after the first may be: a + * Unicode letter (`\p{L}`), decimal digit (`\p{Nd}`), `_`, `$`, or any character whose code is + * `>= 0x80`. The `>= 0x80` part is a code-point range, not a per-`Char` one, so a supplementary- + * plane character written as a surrogate pair in a Kotlin `String` matches as the single code + * point it represents, not as two separate units. */ internal const val COLUMN_REFERENCE_IDENTIFIER_CONTINUATION = """[\p{L}\p{Nd}_$\x{80}-\x{10FFFF}]""" @@ -105,29 +70,17 @@ private const val COLUMN_REFERENCE_IDENTIFIER = """$COLUMN_REFERENCE_IDENTIFIER_START$COLUMN_REFERENCE_IDENTIFIER_CONTINUATION*""" /** - * Matches a double-quoted PostgreSQL identifier, quotes included — `"` followed by any number of - * (a non-`"` character) or (a doubled `""`, PostgreSQL's escape for a literal `"` inside the - * name), followed by the closing `"`. Not unescaped by this pattern itself — that is - * [unescapeQuotedIdentifier]'s job, once a caller has the matched raw token (quotes and any - * doubled escapes still intact) in hand. - * - * A separate constant from [COLUMN_REFERENCE_IDENTIFIER] (never merged into it): unlike an - * unquoted identifier, a quoted one is legal only as a `table`/`column` position in - * [COLUMN_REFERENCE] — never as a bare function/type name (see [FUNCTION_CALL_START], which - * still uses [COLUMN_REFERENCE_IDENTIFIER_START]/[COLUMN_REFERENCE_IDENTIFIER_CONTINUATION] - * directly and is unaffected by this constant). + * Matches a double-quoted PostgreSQL identifier, quotes included: `"` followed by any number of a + * non-`"` character or a doubled `""` (PostgreSQL's escape for a literal `"` inside the name), + * followed by the closing `"`. Not unescaped by this pattern itself — that is + * [unescapeQuotedIdentifier]'s job. */ private const val QUOTED_IDENTIFIER = "\"(?:[^\"]|\"\")*\"" /** - * [QUOTED_IDENTIFIER] compiled once, for callers that need to find/match a quoted identifier - * token starting at a known position within a larger string — [Regex.matchAt] — rather than - * matching an entire already-isolated string the way [COLUMN_REFERENCE] does via - * [Regex.matchEntire]. [parseAliasToken] is the one caller: it needs the escape-aware end of a - * quoted alias token starting at a specific index, the same escape handling [COLUMN_REFERENCE] - * already applies via [COLUMN_REFERENCE_IDENTIFIER_OR_QUOTED] — sharing this one compiled - * pattern, rather than writing a second, hand-rolled quote scanner, keeps that handling from - * drifting out of sync between the two call sites. + * [QUOTED_IDENTIFIER] compiled once, for callers that need to match a quoted identifier token + * starting at a known position within a larger string (via [Regex.matchAt]) rather than matching + * an entire already-isolated string (via [Regex.matchEntire], as [COLUMN_REFERENCE] does). */ internal val QUOTED_IDENTIFIER_PATTERN = Regex(QUOTED_IDENTIFIER) @@ -142,26 +95,19 @@ private const val COLUMN_REFERENCE_IDENTIFIER_OR_QUOTED = /** * Matches `table.column` or just `column`, where `table`/`column` are each either an unquoted * PostgreSQL identifier ([COLUMN_REFERENCE_IDENTIFIER_START] followed by zero or more - * [COLUMN_REFERENCE_IDENTIFIER_CONTINUATION] characters — never a bare `\w+`: PostgreSQL's - * identifier class is wider than `\w` (it admits `$` and any `>= 0x80` character — see - * [isIdentifierChar]) but its first character is narrower (`\w+` doesn't enforce that a digit or - * `$` may only appear after the first character)) or a double-quoted one ([QUOTED_IDENTIFIER] — - * `"ux"`, `"My Col"`, `"He""llo"`), matched via [COLUMN_REFERENCE_IDENTIFIER_OR_QUOTED] for each - * position independently. + * [COLUMN_REFERENCE_IDENTIFIER_CONTINUATION] characters) or a double-quoted one + * ([QUOTED_IDENTIFIER] — `"ux"`, `"My Col"`, `"He""llo"`), matched via + * [COLUMN_REFERENCE_IDENTIFIER_OR_QUOTED] for each position independently. * - * The leading-character restriction on the unquoted alternative matters in the widening direction - * specifically: without it, a digit- or `$`-led fragment merely followed by - * identifier-continuation characters — e.g. `2€`, which PostgreSQL itself rejects outright - * ("trailing junk after numeric literal", on PostgreSQL 18.4) — would [Regex.matchEntire] as a - * whole "identifier" once the continuation class widens to admit `€`, handing back a `columnName` - * PostgreSQL would never resolve to that name. [parseColumnReference] returning `null` for - * anything that isn't a real identifier is the safe, intended outcome (see [parseSelectItems]'s - * KDoc on why a lost name degrades safely to `ResultSetMetaData` while a wrong one does not). + * The leading-character restriction on the unquoted alternative matters in the widening + * direction: without it, a digit-led fragment such as `2€` — which PostgreSQL itself rejects + * outright (`trailing junk after numeric literal`, on PostgreSQL 18.4) — would match as a whole + * identifier once the continuation class widens to admit `€`, handing back a name PostgreSQL + * would never resolve to. [parseColumnReference] returns `null` for anything that isn't a real + * identifier. * - * A matched group's captured text still includes its surrounding quotes (if any) — the whole raw - * token, exactly as [QUOTED_IDENTIFIER] defines it — since [Regex] group captures always span - * whatever the sub-pattern matched; [parseColumnReference] turns that raw capture into the - * logical value [SelectItem.columnName]/[SelectItem.tableName] actually store, via + * A matched group's captured text still includes its surrounding quotes (if any); + * [parseColumnReference] turns that raw capture into the logical value via * [unescapeQuotedIdentifier]. */ internal val COLUMN_REFERENCE = Regex( @@ -186,24 +132,46 @@ internal fun unescapeQuotedIdentifier(rawQuotedToken: String): String = /** PostgreSQL's `NAMEDATALEN - 1`: the byte length an identifier is truncated to by the server. */ internal const val MAX_IDENTIFIER_LENGTH_BYTES = 63 +/** + * A PostgreSQL identifier that never needs double-quoting when written back into SQL: starts with + * a lowercase letter or underscore, followed by any number of lowercase letters, digits, + * underscores, or dollar signs. Matching this pattern is necessary but not sufficient — + * [quoteSqlIdentifierIfNeeded] additionally rejects a reserved word, which this pattern alone + * cannot rule out (`order` and `user` both match it). + */ +private val SAFE_UNQUOTED_IDENTIFIER = Regex("[a-z_][a-z0-9_\$]*") + +/** + * Double-quotes [identifier] exactly as PostgreSQL requires it to be written back into SQL — + * doubling any embedded `"` — unless [identifier]'s lowercased form is not one of [reservedWords] + * and it already matches [SAFE_UNQUOTED_IDENTIFIER] bare. + * + * A mixed-case or space-containing name (`"Foo"`, `"My Col"`) rendered bare as `table.Foo` reads + * back as `column tq.foo does not exist`, since PostgreSQL folds `Foo` to `foo`. + * + * A relation or column named after a reserved word (`order`, `user`) rendered bare as `order.id` + * reads back as `syntax error at or near "."`, since an unquoted `order` parses as the reserved + * keyword, not a table reference. [reservedWords] should be the connected server's own live + * keyword set ([JdbcAnalyzer.fetchReservedWords]). + */ +internal fun quoteSqlIdentifierIfNeeded(identifier: String, reservedWords: Set): String = + if (identifier.lowercase() in reservedWords || !identifier.matches(SAFE_UNQUOTED_IDENTIFIER)) { + "\"${identifier.replace("\"", "\"\"")}\"" + } else { + identifier + } + /** * Truncates [identifier] the way PostgreSQL does when it reaches the server * (`downcase_truncate_identifier` in `scan.l`): to the longest prefix of at most * [MAX_IDENTIFIER_LENGTH_BYTES] UTF-8 bytes, dropping whole characters rather than splitting one. * A 32-character name of `é` is 64 bytes and truncates to 31 characters, not to a broken 63rd byte. * - * The server applies this to references as well as declarations, so `SELECT <70-char name>` - * resolves against a column created with that same name — both sides were already truncated to the - * same 63 bytes before being compared. Norm has to do the same to any name it reads out of SQL text - * or build configuration, since everything it reads from the server arrives truncated already. - * * [identifier] must be the logical value, with quotes and any `""` escape already resolved; this - * only measures bytes and has no opinion on quoting. Truncating a still-quoted name would count the - * quote characters and can drop the closing one. + * only measures bytes and has no opinion on quoting. Truncating a still-quoted name would count + * the quote characters and can drop the closing one. * - * Not part of [foldIdentifier], despite both being identifier-comparison rules: `parse_ident()` - * does not truncate, and `JdbcAnalyzerTest`'s `FoldIdentifierParseIdentDifferentialTest` pins - * [foldIdentifier] against it. + * Not part of [foldIdentifier]: `parse_ident()` does not truncate. */ internal fun truncateIdentifier(identifier: String): String { var byteLength = 0 diff --git a/generator/src/main/kotlin/norm/generator/SqlLexer.kt b/generator/src/main/kotlin/norm/generator/SqlLexer.kt index 7fa0fd84..64f8a98f 100644 --- a/generator/src/main/kotlin/norm/generator/SqlLexer.kt +++ b/generator/src/main/kotlin/norm/generator/SqlLexer.kt @@ -38,15 +38,8 @@ internal fun stripComments(text: String): String { /** * True if [character] can appear inside an unquoted PostgreSQL identifier, at any position after * the first: a letter, digit, underscore, dollar sign, or any character whose code is `>= 0x80`. - * PostgreSQL's `scan.l` `ident_cont` class is wider than a plain letter/digit/`_`/`$` check — it - * also admits combining marks, currency and other symbols (`€`, `©`, `¹`), and supplementary-plane - * characters (a UTF-16 surrogate pair, both units `>= 0x80`) that Kotlin's `isLetterOrDigit()` - * alone doesn't recognize. * - * Every keyword word-boundary check and identifier-run scan in this file shares this one - * predicate, so they always agree on where an identifier ends — otherwise a PostgreSQL-legal - * `>= 0x80` character could truncate a run at the wrong place (e.g. `returning€`, a legal column - * name, misread as the bare keyword `RETURNING` plus a stray `€`). + * `returning€` is a legal PostgreSQL column name, not the keyword `RETURNING` plus a stray `€`. */ internal fun isIdentifierChar(character: Char): Boolean = character.isLetterOrDigit() || character == '_' || character == '$' || character.code >= 0x80 @@ -56,13 +49,8 @@ internal fun isIdentifierChar(character: Char): Boolean = * whose code is `>= 0x80` — never a digit or `$`, both of which are legal only after the first * character (see [isIdentifierChar]). * - * Also the predicate for what can start a dollar-quote tag (the `tag` in `$tag$...$tag$`): per - * PostgreSQL's `scan.l`, `ident_start` and `dolq_start` are defined by the identical character - * class, `[A-Za-z\200-\377_]` — a genuine coincidence, not an approximation. The two continuation - * classes diverge instead (`ident_cont` admits `$`; `dolq_cont` admits digits but never `$` — see - * [isDollarQuoteTagContinuationChar]), which is why [isIdentifierChar] and - * [isDollarQuoteTagContinuationChar] stay separate predicates even though the start class is - * shared here. + * Also the predicate for what can start a dollar-quote tag (the `tag` in `$tag$...$tag$`): + * PostgreSQL uses the identical character class for both. */ internal fun isIdentifierStartChar(character: Char): Boolean = character.isLetter() || character == '_' || character.code >= 0x80 @@ -72,10 +60,6 @@ internal fun isIdentifierStartChar(character: Char): Boolean = * PostgreSQL allows inside an identifier (see [isIdentifierChar]), so it isn't merely a prefix * of a longer identifier — advances past it and any trailing whitespace/comments. Otherwise * returns [position] unchanged. - * - * A comment immediately abutting the keyword (`WHEN NOT/*c*/MATCHED`) is a valid separator: `/` - * is not an identifier character, so the boundary check accepts it, and the trailing - * `skipWhitespaceAndComments` call then advances past the comment itself. */ internal fun skipOptionalKeyword(sql: String, position: Int, keyword: String): Int { if (!sql.regionMatches(position, keyword, 0, keyword.length, ignoreCase = true)) return position @@ -141,29 +125,21 @@ internal fun skipBlockComment(sql: String, start: Int): Int { * If `sql[position]` begins a lexical token that character-by-character scanners in this file * must treat as an opaque unit — a single-quoted string literal (`E'...'` escape strings, * `''`-doubled quotes), a double-quoted identifier (`""`-doubled quotes), a dollar-quoted string - * (`$$...$$` or `$tag$...$tag$`, but only when the `$` is not itself continuing an identifier — - * see the dollar-quote branch below and [skipDollarQuotedString]'s KDoc; PostgreSQL allows `$` - * inside an unquoted identifier, so `a$b$c` is one identifier, not a dollar-quote-delimited - * string starting after `a`), a `--` line comment, or a `/* */` block comment — returns the index - * immediately after that token. Otherwise returns [position] unchanged, meaning the caller should - * process this character itself (as a keyword character, a parenthesis, a delimiter, etc.). + * (`$$...$$` or `$tag$...$tag$`, but only when the `$` is not itself continuing an identifier), + * a `--` line comment, or a `/* */` block comment — returns the index immediately after that + * token. Otherwise returns [position] unchanged, meaning the caller should process this character + * itself (as a keyword character, a parenthesis, a delimiter, etc.). * - * This is the single place that understands enough of SQL's lexical structure to keep the raw - * text scanners in this file ([findTopLevelKeyword], [findMatchingCloseParenthesis], - * `splitAtTopLevel`, `extractAlias`, and any other paren-depth or keyword search) from - * misreading a `(`, `)`, or keyword that only appears inside a string, a quoted identifier, or a - * comment — e.g. `RETURNING regexp_replace(name, '\(', '')` has an unbalanced `(` inside its - * string literal, and `SET name = 'copied from source'` has the word `from` inside a string - * literal, neither of which is a real paren or keyword. Every such scanner calls this at each - * position and jumps ahead when it returns a different index, rather than inspecting - * `sql[position]` directly. + * Every paren-depth or keyword search in this file calls this at each position and jumps ahead + * when it returns a different index, rather than inspecting `sql[position]` directly — so a `(` + * or keyword that only appears inside a string, a quoted identifier, or a comment is never + * misread as a real one (`RETURNING regexp_replace(name, '\(', '')` has an unbalanced `(` inside + * its string literal). * * @param adjacency See [OriginalAdjacency]'s KDoc. Defaults to [ALL_ADJACENT], correct for raw SQL * text; [StrippedText] threads itself here for its own [StrippedText.skipLexicalToken] entry - * point, gating the `--` line-comment and `/* */` block-comment openers and the `$` dollar-quote - * identifier lookback below (and, transitively, the standalone-`E` and `''`/`""` doubled-quote - * checks inside [skipSingleQuotedString]/[skipDoubleQuotedIdentifier]) on whether stripping - * actually fused these characters together, versus PostgreSQL itself having lexed them adjacent. + * point, gating the checks below on whether stripping actually fused these characters together, + * versus PostgreSQL itself having lexed them adjacent. * @return The index after the lexical token, or [position] if none starts there. */ internal fun skipLexicalToken(sql: String, position: Int, adjacency: OriginalAdjacency = ALL_ADJACENT): Int { @@ -171,23 +147,12 @@ internal fun skipLexicalToken(sql: String, position: Int, adjacency: OriginalAdj return when { sql[position] == '\'' -> skipSingleQuotedString(sql, position, adjacency) sql[position] == '"' -> skipDoubleQuotedIdentifier(sql, position, adjacency) - // A "$" immediately after an identifier character (e.g. the second "$" in "a$b$c", or - // either "$" in "x$$y") can't open a dollar quote — it continues the identifier that - // started before it, since PostgreSQL's own lexer only recognizes dollar-quote tags at the - // start of a new token. Without this guard, "a$b$c" reads as "a" followed by a "$b$"-tagged - // dollar-quote opener that swallows everything up to the next literal "$b$" (or the rest of - // the string, if there isn't one). - // - // Gated on adjacency.wereAdjacent(position - 1): stripping only ever removes whitespace and - // comments, neither of which is an identifier character, so if the character now sitting - // immediately before this "$" was not actually adjacent to it in the original text, this "$" - // genuinely opens a new token regardless of what character stripping fused in front of it - // (e.g. "x $q$...$q$" strips to "x$q$...$q$", where the "x" never actually continued into - // "$q$" in the original query). + // A "$" immediately after an identifier character (e.g. the second "$" in "a$b$c") can't open + // a dollar quote -- it continues the identifier that started before it. Gated on + // adjacency.wereAdjacent(position - 1): "x $q$...$q$" strips to "x$q$...$q$", where "x" never + // actually continued into "$q$" in the original query. sql[position] == '$' && !(position > 0 && adjacency.wereAdjacent(position - 1) && isIdentifierChar(sql[position - 1])) -> - // skipDollarQuotedString takes the same adjacency and applies its own further gates to the - // opening and closing delimiters themselves — see its KDoc. skipDollarQuotedString(sql, position, adjacency) ?: position sql[position] == '-' && position + 1 < sql.length && sql[position + 1] == '-' && adjacency.wereAdjacent(position) -> skipLineComment(sql, position) @@ -205,22 +170,14 @@ internal fun skipLexicalToken(sql: String, position: Int, adjacency: OriginalAdj * always consumes the following character as a literal, so it can never end the string). * * The "standalone" check — the character before that `E`/`e`, if any, is not itself a letter, - * digit, or `_` — uses [isIdentifierChar], the same predicate every other word-boundary check in - * this file uses, gated on [adjacency]: [isStarItem] normalizes a select item through - * [stripCommentsAndWhitespace] and then re-lexes the stripped string, so a lookback using the - * full [isIdentifierChar] class (which admits `$` and any `>= 0x80` character, both legal - * PostgreSQL identifier-continuation characters that a narrower letter/digit/`_` check would - * miss) must be gated on whether the character immediately before `E`/`e` was genuinely adjacent - * in the original text — otherwise a separator stripping removed (e.g. the space in - * `x€ E'a\'b'`) could fuse into `E` and manufacture a standalone-`E` escape string match that was - * never in the query, mis-lexing a valid typed-literal call (`x E'a\'b'`, PostgreSQL's - * `AexprConst: func_name Sconst` form) as something else entirely. + * digit, or `_` — uses [isIdentifierChar], gated on [adjacency]: a stripped-away separator + * (e.g. the space in `x€ E'a\'b'`) could otherwise fuse into `E` and manufacture a + * standalone-`E` escape string match that was never in the query, mis-lexing a valid + * identifier-then-string-literal (`x E'a\'b'`) as something else entirely. * * @param adjacency See [OriginalAdjacency]'s KDoc. Gates both the "is the character immediately * before the opening quote genuinely `E`/`e`" check and, when it is, the standalone lookback one - * position further back — non-adjacency at that second position means the real predecessor was a - * separator PostgreSQL itself lexed on, so `E`/`e` is standalone regardless of what character - * stripping fused in front of it. + * position further back. * @return The index after the closing quote, or `sql.length` if unterminated. */ private fun skipSingleQuotedString(sql: String, openQuoteIndex: Int, adjacency: OriginalAdjacency = ALL_ADJACENT): Int { @@ -239,11 +196,8 @@ private fun skipSingleQuotedString(sql: String, openQuoteIndex: Int, adjacency: if (sql[i] == '\'') { // The '' doubled-quote-escape check is gated on adjacency too: if a separator PostgreSQL // lexed between two genuinely separate quote characters gets stripped away, fusing them - // into what looks like a doubled '' escape, treating it as one would keep this scan going - // (still in isEscapeString mode, if it started that way) past what should have been the - // first string's real terminator — potentially overrunning all the way to sql.length and - // defeating findTrailingImplicitAliasStart's "last segment must end exactly at - // text.length" anchor (see its KDoc) in the dangerous direction (see isStarItem's KDoc). + // into what looks like a doubled '' escape, treating it as one would overrun past what + // should have been the first string's real terminator, all the way to sql.length. val firstQuoteIndex = i i++ if (i < sql.length && sql[i] == '\'' && adjacency.wereAdjacent(firstQuoteIndex)) { @@ -292,26 +246,19 @@ internal fun skipDoubleQuotedIdentifier( } /** - * True if [character] can continue a dollar-quote tag after its first character, per - * PostgreSQL's `scan.l`: `dolq_cont = [A-Za-z\200-\377_0-9]`. A tag's first character is instead - * gated by [isIdentifierStartChar] — per `scan.l`, `dolq_start` and `ident_start` are the - * identical class, so this file shares one predicate for both (see [isIdentifierStartChar]'s - * KDoc). This continuation predicate does not admit `$` (unlike [isIdentifierChar]), since `$` - * delimits the tag rather than continuing it — the one place `scan.l` splits the two kinds of run - * apart, which is why continuation stays a separate predicate even though start does not. It does - * admit digits, unlike a tag's first character (a tag may not start with one — PostgreSQL rejects - * `$1$foo$1$` as a dollar-quoted string entirely, leaving the `$1` to be read as an ordinary, - * non-quote `$`-prefixed token instead). + * True if [character] can continue a dollar-quote tag after its first character: a letter, digit, + * underscore, or any character whose code is `>= 0x80`. A tag's first character is instead gated + * by [isIdentifierStartChar], which does not admit digits: a tag may not start with one — + * PostgreSQL rejects `$1$foo$1$` as a dollar-quoted string entirely, leaving the `$1` to be read + * as an ordinary, non-quote `$`-prefixed token instead. */ private fun isDollarQuoteTagContinuationChar(character: Char): Boolean = character.isLetterOrDigit() || character == '_' || character.code >= 0x80 /** * `true` if every consecutive pair of characters in `[start, endExclusive)` was genuinely - * adjacent in the original text, per [adjacency] — see [OriginalAdjacency]'s KDoc. Used by - * [skipDollarQuotedString] to verify its opening delimiter is lexically contiguous, not merely - * contiguous in [stripCommentsAndWhitespace]'s stripped output — see that function's KDoc for why - * the closing delimiter needs no such check of its own. + * adjacent in the original text, per [adjacency]. Used by [skipDollarQuotedString] to verify its + * opening delimiter is lexically contiguous, not merely contiguous in a stripped-and-fused string. */ private fun isAdjacencyContiguousSpan(adjacency: OriginalAdjacency, start: Int, endExclusive: Int): Boolean { for (leftIndex in start until endExclusive - 1) { @@ -326,41 +273,22 @@ private fun isAdjacencyContiguousSpan(adjacency: OriginalAdjacency, start: Int, * [isDollarQuoteTagContinuationChar] characters — advances past the matching closing tag (the * same `$$`/`$tag$` again). * - * Callers must first confirm [position] is not immediately preceded by an identifier character - * (see [skipLexicalToken]'s call site) — this function has no way to tell, from `$` alone, - * whether it is looking at a genuine dollar-quote opener or the second `$` of an ordinary - * identifier like `a$b$c` (PostgreSQL allows `$` inside an unquoted identifier), so that - * decision is made by the caller before this is even invoked. + * Callers must first confirm [position] is not immediately preceded by an identifier character — + * this function has no way to tell, from `$` alone, whether it is looking at a genuine + * dollar-quote opener or the second `$` of an ordinary identifier like `a$b$c`. * * The opening delimiter is additionally required to be adjacency-contiguous (see - * [OriginalAdjacency]'s KDoc): every character of it — the leading `$` at [position], the tag run, - * and the tag's closing `$` — must have been genuinely adjacent to its neighbour in the original - * text. Without this, stripping can invent a delimiter that was never one lexical unit in the - * original query — e.g. `$q b $/ /` (two independent `$`-prefixed tokens and a `/`-prefixed - * token, none of them a real dollar-quote) strips to `$qb$//`, whose fused `$qb$` would otherwise - * be read as an opening delimiter with tag `qb`, and `//` as its (unterminated) body. - * - * The closing delimiter needs no adjacency check of its own: once the opening delimiter has passed - * both this gate and the caller's own `$`-not-preceded-by-an-identifier-character lookback (see - * [skipLexicalToken]'s call site), the original text genuinely opened a dollar-quoted string at - * [position] — and [stripCommentsAndWhitespace] makes that identical determination (via this same - * function, on the original text) before ever stripping anything, so it copies the entire matched - * token — opening delimiter, body, and closing delimiter alike — through to its output verbatim, - * giving every character in that span consecutive original offsets. A real closing tag inside a - * verbatim-copied span is therefore always adjacency-contiguous already, by construction, and no - * fused, fake closing tag can appear inside one either. An unterminated dollar-quote — no closing - * tag found in [sql] at all — is unaffected by any of this: it is not a stripping artifact, and is - * handled the same way regardless. + * [OriginalAdjacency]'s KDoc): every character of it must have been genuinely adjacent to its + * neighbour in the original text. Without this, stripping can invent a delimiter that was never + * one lexical unit in the original query — e.g. `$q b $/ /` strips to `$qb$//`, whose fused + * `$qb$` would otherwise be read as an opening delimiter with tag `qb`. * * @param adjacency See [OriginalAdjacency]'s KDoc. Defaults to [ALL_ADJACENT], correct for raw SQL - * text — every neighbouring pair is trivially adjacent there, so this gate never rejects a - * genuine raw-text dollar-quote. [skipLexicalToken] threads its own `adjacency` parameter through - * here. + * text. * @return The index after the closing tag (or `sql.length` if unterminated), or `null` if * [position] is a `$` that is not followed by a valid closing tag delimiter at all (e.g. a * bare `$` used as an operator, or a positional parameter marker like `$1` with no matching - * second `$`), or if the opening delimiter itself is not adjacency-contiguous — the caller - * should treat that `$` as an ordinary character, not lexically skip it. + * second `$`), or if the opening delimiter itself is not adjacency-contiguous. */ private fun skipDollarQuotedString(sql: String, position: Int, adjacency: OriginalAdjacency = ALL_ADJACENT): Int? { var i = position + 1 @@ -377,3 +305,45 @@ private fun skipDollarQuotedString(sql: String, position: Int, adjacency: Origin val closeIndex = sql.indexOf(closingTag, openingTagEnd) return if (closeIndex < 0) sql.length else closeIndex + closingTag.length } + +/** + * Collapses the cosmetic whitespace [stripComments]' own single-space substitution can leave behind + * in an expression about to be embedded verbatim in generated KDoc — a comment directly after an + * opening parenthesis or before a closing one (`UPPER(/* x */a)` strips to `UPPER( a)`) reads oddly + * there. Applied only where an expression is resolved for KDoc, never inside [stripComments] itself. + * + * Collapses whitespace outside a single-quoted literal, a dollar-quoted string, a quoted identifier, + * or a comment to a single space, then removes a single such space immediately after `(` or before + * `)` — never semantically significant in SQL. + * + * Walks every span verbatim via [skipLexicalToken] rather than a whitespace-collapse regex, which + * can't tell a cosmetic space from one inside the developer's own SQL and would rewrite a quoted + * identifier's internal spacing (`"My Col"` to `"My Col"`, a column name PostgreSQL then rejects) + * or a string literal's contents (`'( x )'` to `'(x)'`). + */ +internal fun collapseCosmeticWhitespace(text: String): String { + val trimmed = text.trim() + val builder = StringBuilder(trimmed.length) + var i = 0 + while (i < trimmed.length) { + val afterToken = skipLexicalToken(trimmed, i) + if (afterToken != i) { + builder.append(trimmed, i, afterToken) + i = afterToken + continue + } + val character = trimmed[i] + if (!character.isWhitespace()) { + builder.append(character) + i++ + continue + } + var afterWhitespace = i + while (afterWhitespace < trimmed.length && trimmed[afterWhitespace].isWhitespace()) afterWhitespace++ + val precededByOpenParenthesis = builder.isNotEmpty() && builder.last() == '(' + val followedByCloseParenthesis = afterWhitespace < trimmed.length && trimmed[afterWhitespace] == ')' + if (!precededByOpenParenthesis && !followedByCloseParenthesis) builder.append(' ') + i = afterWhitespace + } + return builder.toString() +} diff --git a/generator/src/main/kotlin/norm/generator/SqlMappable.kt b/generator/src/main/kotlin/norm/generator/SqlMappable.kt index 2c4856c0..f23511f1 100644 --- a/generator/src/main/kotlin/norm/generator/SqlMappable.kt +++ b/generator/src/main/kotlin/norm/generator/SqlMappable.kt @@ -7,19 +7,10 @@ import com.squareup.kotlinpoet.MemberName import com.squareup.kotlinpoet.ParameterizedTypeName.Companion.parameterizedBy import com.squareup.kotlinpoet.TypeName import com.squareup.kotlinpoet.asTypeName -import java.math.BigDecimal -import java.sql.Blob -import java.sql.ResultSet -import java.sql.Statement import java.sql.Types import java.time.Instant -import java.time.LocalDate -import java.time.LocalDateTime -import java.time.LocalTime import java.time.OffsetDateTime -import java.time.OffsetTime import java.time.ZoneOffset -import kotlin.reflect.KClass /** * Column index of the `VALUE` column in the element `ResultSet` returned by @@ -32,69 +23,246 @@ private const val ELEMENT_VALUE_COLUMN_INDEX = 2 */ internal interface SqlMappable { - /** - * Kotlin [KClass] for the data. - */ - val klass: KClass<*> - /** * KotlinPoet [TypeName] for the data. */ val typeName: TypeName - get() = klass.asTypeName() /** - * Receiver action to call on a [Statement] when mapping the data from Java to SQL. + * Receiver action to call on a [Statement][java.sql.Statement] when mapping the data from Java + * to SQL. */ val statementAction: (index: Int, parameterName: CodeBlock) -> CodeBlock /** - * Receiver action to call on a [ResultSet] when mapping the data from SQL to Java. + * Receiver action to call on a [ResultSet][java.sql.ResultSet] when mapping the data from SQL + * to Java. */ val resultSetAction: (index: Int) -> CodeBlock } /** - * Types with first-class support in JDBC. + * Wire-level JDBC access for a Postgres base type: which `ResultSet`/`PreparedStatement` methods + * read and write it, and the Kotlin type JDBC delivers it as. Used both for a plain (adapterless) + * column ([ScalarSqlMappable]) and for the same type behind a `norm.ColumnAdapter` + * ([AdaptedTypeSqlMappable]) — a domain, an enum, or a user-configured type mapping. */ -internal enum class JdbcTypes(override val klass: KClass<*>) : SqlMappable { - BOOLEAN(Boolean::class), - SHORT(Short::class), - INT(Int::class), - LONG(Long::class), - FLOAT(Float::class), - DOUBLE(Double::class), - BIG_DECIMAL(BigDecimal::class), - STRING(String::class), - BLOB(Blob::class), - ; - - override val statementAction: (Int, CodeBlock) -> CodeBlock = - { index, parameterName -> CodeBlock.of("%N(%L, %L)", "set${klass.simpleName}", index, parameterName) } - override val resultSetAction: (Int) -> CodeBlock = - { index -> CodeBlock.of("%N(%L)", "get${klass.simpleName}", index) } +internal interface WireCodec { + + /** + * The non-null Kotlin type JDBC delivers this value as (e.g. `String` for text/varchar, `Int` + * for int4). + */ + val kotlinType: TypeName /** - * See [NullablePrimitiveDecorator]. + * Reads the value at [index]. When [nullable], the rendered expression itself handles a SQL + * `NULL` (returning Kotlin `null`); when not, it assumes the column is `NOT NULL`. */ - fun decorateForNullable(notNull: Boolean): SqlMappable = if (notNull) this else NullablePrimitiveDecorator(this) + fun read(index: Int, nullable: Boolean): CodeBlock + + /** + * Writes a non-null [value] at [index]. + */ + fun write(index: Int, value: CodeBlock): CodeBlock + + /** + * Writes SQL `NULL` at [index]. + */ + fun writeNull(index: Int): CodeBlock + + /** + * Writes a value at [index] that may be `null` at runtime. + * + * Defaults to [write]: most codecs' JDBC setter already accepts and forwards a `null` argument + * correctly (`setObject`, or a plain named setter whose Postgres-side coercion handles `NULL`), + * so no extra branching is needed. [PrimitiveCodec] overrides this — a JVM primitive setter + * cannot accept `null` at all — and is the only override; changing any other codec's default + * here would regenerate goldens for `text`, `numeric`, `oid`, `bytea`, `date`, `time`, `timetz`, + * `timestamp`, `uuid`, `json`, and `jsonb` plain nullable columns. + */ + fun writeNullable(index: Int, value: CodeBlock): CodeBlock = write(index, value) +} + +/** + * [WireCodec] for a JVM primitive delivered through a named getter/setter pair (`getInt`/`setInt`, + * etc.). JDBC getters for primitives return `0`/`false` rather than `null` for a SQL `NULL`, so a + * nullable read needs a `wasNull()` check; a JVM primitive setter cannot accept `null` at all, so a + * nullable write goes through a `norm.set` runtime extension (which accepts a nullable argument + * and calls `setNull` itself when it is `null`) instead of the plain setter. + * + * @param methodName The JDBC method name suffix shared by the getter/setter pair (e.g. `"Int"` for + * `getInt`/`setInt`). + * @param sqlTypeConstant The field name on [java.sql.Types] for `setNull()` calls (e.g. + * `"INTEGER"`). + */ +internal class PrimitiveCodec( + override val kotlinType: TypeName, + private val methodName: String, + private val sqlTypeConstant: String, +) : WireCodec { + + override fun read(index: Int, nullable: Boolean): CodeBlock { + val get = CodeBlock.of("%N(%L)", "get$methodName", index) + return if (nullable) CodeBlock.of("%L.takeUnless { wasNull() }", get) else get + } + + override fun write(index: Int, value: CodeBlock): CodeBlock = + CodeBlock.of("%N(%L, %L)", "set$methodName", index, value) + + override fun writeNull(index: Int): CodeBlock = + CodeBlock.of("setNull(%L, %T.%N)", index, Types::class, sqlTypeConstant) + + override fun writeNullable(index: Int, value: CodeBlock): CodeBlock { + val member = MemberName("norm", "set$methodName", isExtension = true) + return CodeBlock.of("%M(%L, %L)", member, index, value) + } +} + +/** + * [WireCodec] for a non-primitive type delivered through a named getter/setter pair + * (`getString`/`setString`, `getBigDecimal`/`setBigDecimal`, `getBlob`/`setBlob`, + * `getBytes`/`setBytes`) whose declared return/parameter type is already the wire type, and whose + * setter already accepts and forwards `null` correctly. Neither read nor write branches on + * nullability: the getter returns Kotlin `null` for a SQL `NULL` without a `wasNull()` check, and + * the setter accepts a nullable argument directly. + * + * @param sqlTypeConstant The field name on [java.sql.Types] for `setNull()` calls (e.g. + * `"VARCHAR"`). + */ +internal class ObjectGetterCodec( + override val kotlinType: TypeName, + private val getterName: String, + private val setterName: String, + private val sqlTypeConstant: String, +) : WireCodec { + + override fun read(index: Int, nullable: Boolean): CodeBlock = CodeBlock.of("%N(%L)", getterName, index) + + override fun write(index: Int, value: CodeBlock): CodeBlock = CodeBlock.of("%N(%L, %L)", setterName, index, value) + + override fun writeNull(index: Int): CodeBlock = + CodeBlock.of("setNull(%L, %T.%N)", index, Types::class, sqlTypeConstant) +} + +/** + * [WireCodec] for a type bound with `setObject(index, value, Types.OTHER)` rather than a named + * setter — required for Postgres custom/coercion-sensitive types (`json`, `jsonb`, enums) where + * the JDBC driver refuses to coerce a `VARCHAR` binding; `Types.OTHER` bypasses the driver's type + * enforcement and lets Postgres perform the coercion itself. `setObject(index, null, targetSqlType)` + * already delegates to `setNull(index, targetSqlType)`, so, like [ObjectGetterCodec], neither read + * nor write branches on nullability. + * + * @param getterName The `ResultSet` getter method name (always `"getString"` for this codec's + * current uses). + * @param sqlTypeConstant The field name on [java.sql.Types] used both for the `setObject` hint and + * for `setNull()` calls (always `"OTHER"` for this codec's current uses). + */ +internal class TypesOtherCodec( + override val kotlinType: TypeName, + private val getterName: String, + private val sqlTypeConstant: String, +) : WireCodec { + + override fun read(index: Int, nullable: Boolean): CodeBlock = CodeBlock.of("%N(%L)", getterName, index) + + override fun write(index: Int, value: CodeBlock): CodeBlock = + CodeBlock.of("setObject(%L, %L, %T.%N)", index, value, Types::class, sqlTypeConstant) + + override fun writeNull(index: Int): CodeBlock = + CodeBlock.of("setNull(%L, %T.%N)", index, Types::class, sqlTypeConstant) +} + +/** + * [WireCodec] for a type whose read needs the class-qualified `getObject(index, X::class.java)` + * overload rather than a named getter — required whenever the wire type has no dedicated JDBC + * getter: `java.sql.ResultSet.getObject(int)` is declared to return `Object`, so a bare + * `getObject(index)` call is statically `Any` in Kotlin no matter what concrete type the driver + * returns at runtime. Covers the `java.time` types (`LocalDate`, `LocalTime`, `OffsetTime`, + * `LocalDateTime`), where pgjdbc's plain `getObject(int)` returns the legacy + * `java.sql.Date`/`Time`/`Timestamp` even at runtime, and `uuid`, where pgjdbc's plain + * `getObject(int)` does return a `java.util.UUID` at runtime (`PgResultSet.internalGetObject` + * special-cases the Postgres `uuid` type by name) but the static type is still `Any` — the class + * hint is required in both cases, for different reasons (pgjdbc 42.7.13's + * `PgResultSet.getObject(int, Class)` special-cases each of these classes explicitly). + * + * The write side needs no such qualification: `PgPreparedStatement.setObject(int, Object)` already + * dispatches on the runtime type of a `LocalDate`/`LocalTime`/`OffsetTime`/`LocalDateTime`/`UUID` + * argument directly (pgjdbc 42.7.13's source), so `write` is a plain `setObject(index, value)`. + * Like [ObjectGetterCodec], neither read nor write branches on nullability. + * + * @param getterClassHint The class passed to `getObject(index, X::class.java)`; also this codec's + * [kotlinType], since the wire and Kotlin representations are the same type for every use of + * this codec. + * @param sqlTypeConstant The field name on [java.sql.Types] for `setNull()` calls (e.g. `"DATE"`, + * `"OTHER"` for `uuid`). + */ +internal class ClassHintedObjectCodec(private val getterClassHint: ClassName, private val sqlTypeConstant: String) : + WireCodec { + + override val kotlinType: TypeName = getterClassHint + + override fun read(index: Int, nullable: Boolean): CodeBlock = + CodeBlock.of("getObject(%L, %T::class.java)", index, getterClassHint) + + override fun write(index: Int, value: CodeBlock): CodeBlock = CodeBlock.of("setObject(%L, %L)", index, value) + + override fun writeNull(index: Int): CodeBlock = + CodeBlock.of("setNull(%L, %T.%N)", index, Types::class, sqlTypeConstant) +} + +/** + * [WireCodec] for `timestamptz`, whose Kotlin representation ([Instant]) differs from every + * `ResultSet`/`PreparedStatement` call's own wire representation ([OffsetDateTime]) — every other + * codec's wire and Kotlin representations are the same type. + * + * pgjdbc does not support `getObject(i, Instant::class.java)`, so reads go through + * [OffsetDateTime] and convert via `.toInstant()`. Writes convert via + * `OffsetDateTime.ofInstant(value, ZoneOffset.UTC)` before binding. + * + * Unlike [ClassHintedObjectCodec], this requires nullable awareness on both sides: the `.toInstant()` + * chain on a `null` [OffsetDateTime] read would NPE unless guarded by a safe call, and the JVM + * `OffsetDateTime.ofInstant(value, ...)` call would NPE on a `null` [Instant] write unless it takes + * the `writeNullable` `?.let` branch instead. + */ +internal object InstantViaOffsetDateTimeCodec : WireCodec { + + override val kotlinType: TypeName = Instant::class.asTypeName() + + override fun read(index: Int, nullable: Boolean): CodeBlock { + val raw = CodeBlock.of("getObject(%L, %T::class.java)", index, OffsetDateTime::class) + return if (nullable) CodeBlock.of("%L?.toInstant()", raw) else CodeBlock.of("%L.toInstant()", raw) + } + + override fun write(index: Int, value: CodeBlock): CodeBlock = + CodeBlock.of("setObject(%L, %T.ofInstant(%L, %T.UTC))", index, OffsetDateTime::class, value, ZoneOffset::class) + + override fun writeNull(index: Int): CodeBlock = + CodeBlock.of("setNull(%L, %T.TIMESTAMP_WITH_TIMEZONE)", index, Types::class) + + override fun writeNullable(index: Int, value: CodeBlock): CodeBlock = + CodeBlock.of("%L?.let { %L } ?: %L", value, write(index, CodeBlock.of("it")), writeNull(index)) } /** - * Decorates a [SqlMappable] for a primitive value with nullability information. + * [SqlMappable] for a plain (adapterless) column of a Postgres base type, built from its + * [WireCodec]. + * + * @param notNull Whether the column is `NOT NULL`. Controls [typeName] nullability and which of + * [WireCodec.write]/[WireCodec.writeNullable] the write side uses. */ -internal class NullablePrimitiveDecorator(private val delegate: JdbcTypes) : SqlMappable { - override val klass: KClass<*> - get() = delegate.klass +internal class ScalarSqlMappable(private val codec: WireCodec, private val notNull: Boolean) : SqlMappable { + override val typeName: TypeName - get() = delegate.typeName.copy(true) + get() = codec.kotlinType.copy(nullable = !notNull) + override val statementAction: (index: Int, parameterName: CodeBlock) -> CodeBlock get() = { index, parameterName -> - val member = MemberName("norm", "set${klass.simpleName}", isExtension = true) - CodeBlock.of("%M(%L, %L)", member, index, parameterName) + if (notNull) codec.write(index, parameterName) else codec.writeNullable(index, parameterName) } + override val resultSetAction: (index: Int) -> CodeBlock - get() = { CodeBlock.of("%L.takeUnless { wasNull() }", delegate.resultSetAction(it)) } + get() = { index -> codec.read(index, !notNull) } } /** @@ -122,9 +290,6 @@ internal class ArrayTypeDecorator( private val toSqlArrayMember = MemberName("norm", "toSqlArray", isExtension = true) private val mapElementsMember = MemberName("norm", "mapElements", isExtension = true) - override val klass: KClass<*> - get() = delegate.klass - override val typeName: TypeName get() = arrayTypeName @@ -175,204 +340,14 @@ internal class ArrayTypeDecorator( } } -/** - * Types with support in the Postgres JDBC driver. - */ -internal enum class PostgresSupportedTypes( - override val klass: KClass<*>, - override val statementAction: (Int, CodeBlock) -> CodeBlock, - override val resultSetAction: (index: Int) -> CodeBlock, -) : SqlMappable { - UUID( - java.util.UUID::class, - { index, parameterName -> CodeBlock.of("setObject(%L, %L)", index, parameterName) }, - { index -> CodeBlock.of("getObject(%L, %T::class.java)", index, java.util.UUID::class) }, - ), - LOCAL_DATE( - LocalDate::class, - { index, parameterName -> CodeBlock.of("setObject(%L, %L)", index, parameterName) }, - { index -> CodeBlock.of("getObject(%L, %T::class.java)", index, LocalDate::class) }, - ), - LOCAL_TIME( - LocalTime::class, - { index, parameterName -> CodeBlock.of("setObject(%L, %L)", index, parameterName) }, - { index -> CodeBlock.of("getObject(%L, %T::class.java)", index, LocalTime::class) }, - ), - OFFSET_TIME( - OffsetTime::class, - { index, parameterName -> CodeBlock.of("setObject(%L, %L)", index, parameterName) }, - { index -> CodeBlock.of("getObject(%L, %T::class.java)", index, OffsetTime::class) }, - ), - LOCAL_DATE_TIME( - LocalDateTime::class, - { index, parameterName -> CodeBlock.of("setObject(%L, %L)", index, parameterName) }, - { index -> CodeBlock.of("getObject(%L, %T::class.java)", index, LocalDateTime::class) }, - ), - BYTE_ARRAY( - ByteArray::class, - { index, parameterName -> CodeBlock.of("setBytes(%L, %L)", index, parameterName) }, - { index -> CodeBlock.of("getBytes(%L)", index) }, - ), -} - -/** - * [SqlMappable] for `timestamptz` columns mapped to [Instant]. - * - * pgjdbc does not support `getObject(i, Instant::class.java)`, so reads go through - * [OffsetDateTime] and convert via `toInstant()`. Writes convert via - * `OffsetDateTime.ofInstant(value, ZoneOffset.UTC)`. - * - * Unlike [PostgresSupportedTypes] entries, this requires nullable awareness because the - * `.toInstant()` chain on a null [OffsetDateTime] would NPE. Other [PostgresSupportedTypes] - * entries return Java platform types directly, so null propagates naturally. - */ -internal class InstantSqlMappable(private val notNull: Boolean) : SqlMappable { - - override val klass: KClass<*> = Instant::class - - override val typeName: TypeName - get() = klass.asTypeName().copy(nullable = !notNull) - - override val statementAction: (index: Int, parameterName: CodeBlock) -> CodeBlock - get() = if (notNull) { - { index, parameterName -> - CodeBlock.of( - "setObject(%L, %T.ofInstant(%L, %T.UTC))", - index, - OffsetDateTime::class, - parameterName, - ZoneOffset::class, - ) - } - } else { - { index, parameterName -> - CodeBlock.of( - "%L?.let { setObject(%L, %T.ofInstant(it, %T.UTC)) } ?: setNull(%L, %T.TIMESTAMP_WITH_TIMEZONE)", - parameterName, - index, - OffsetDateTime::class, - ZoneOffset::class, - index, - Types::class, - ) - } - } - - override val resultSetAction: (index: Int) -> CodeBlock - get() = if (notNull) { - { index -> - CodeBlock.of( - "getObject(%L, %T::class.java).toInstant()", - index, - OffsetDateTime::class, - ) - } - } else { - { index -> - CodeBlock.of( - "getObject(%L, %T::class.java)?.toInstant()", - index, - OffsetDateTime::class, - ) - } - } -} - -/** - * [SqlMappable] for plain (adapterless) `json` and `jsonb` columns. - * - * The Postgres JDBC driver rejects `setString()` for both types in prepared statements - * (`column "..." is of type jsonb but expression is of type character varying`), exactly as it does - * for enum columns. Binding with `setObject(index, value, Types.OTHER)` sends the value with an - * unspecified OID and lets Postgres perform the coercion. `json` and `jsonb` differ only in storage - * and in what Postgres preserves on the way in, not in how a parameter is bound or read, so both - * share this mapping. - * - * Unlike [InstantSqlMappable], the write path does not branch on nullability: pgjdbc's - * `setObject(index, null, targetSqlType)` delegates to `setNull(index, targetSqlType)`, so one - * code path covers both cases. - * - * Reads use `getString`, which works for both types and returns `null` for SQL `NULL`. - * - * Keep in sync with the `"json"` and `"jsonb"` entries in [resolveJdbcTypeInfo], which define the - * same binding for the adapter path (user-configured `json`/`jsonb` type mappings and domains built - * on them). - * - * @param notNull Whether the column is `NOT NULL`. Affects [typeName] nullability only. - */ -internal class JsonSqlMappable(private val notNull: Boolean) : SqlMappable { - - override val klass: KClass<*> = String::class - - override val typeName: TypeName - get() = klass.asTypeName().copy(nullable = !notNull) - - override val statementAction: (index: Int, parameterName: CodeBlock) -> CodeBlock - get() = { index, parameterName -> - CodeBlock.of("setObject(%L, %L, %T.OTHER)", index, parameterName, Types::class) - } - - override val resultSetAction: (index: Int) -> CodeBlock - get() = { index -> CodeBlock.of("getString(%L)", index) } -} - -/** - * JDBC method metadata for a type's wire representation, used to generate the correct - * `ResultSet` and `PreparedStatement` calls for reading and writing values through an adapter. - * - * @property getterName The `ResultSet` getter method name (e.g., `"getString"`, `"getInt"`). - * @property setterName The `PreparedStatement` setter method name (e.g., `"setString"`, `"setInt"`). - * @property isPrimitive Whether the JDBC getter returns a JVM primitive (`true` for `Int`, `Short`, - * `Long`, `Float`, `Double`, `Boolean`). Primitives require a `wasNull()` check for nullable columns - * because JDBC returns `0`/`false` instead of `null`. - * @property sqlTypeConstant The field name on [java.sql.Types] for `setNull()` calls (e.g., `"VARCHAR"`, `"INTEGER"`). - * @property useSqlTypeHint When `true`, the setter is generated as `setObject(index, value, Types.sqlTypeConstant)` - * instead of `setterName(index, value)`. Required for Postgres custom types (enums) where the JDBC driver - * refuses to coerce a `VARCHAR` binding — passing `Types.OTHER` bypasses the driver's type enforcement - * and lets Postgres perform the coercion itself. - * @property kotlinType The KotlinPoet [TypeName] for the Kotlin type that JDBC delivers this value as - * (e.g., `String` for text/varchar, `Int` for int4). This is the wire type used for adapter type parameters - * and domain value class properties. - * @property getterClassHint When non-`null`, the read is generated as `getObject(index, X::class.java)` - * (where `X` is [getterClassHint]) instead of `getterName(index)`. Required whenever [getterName] is - * the generic `"getObject"`: `java.sql.ResultSet.getObject(int)` is declared to return `Object`, so a - * bare `getObject(index)` call is statically `Any` in Kotlin no matter what concrete type the driver - * returns at runtime, and that `Any` cannot be passed to a `ColumnAdapter.decode` - * expecting [kotlinType]. This covers both the `java.time` types (`LocalDate`, `LocalTime`, - * `OffsetTime`, `LocalDateTime`, `OffsetDateTime`), where pgjdbc's plain `getObject(int)` returns the - * legacy `java.sql.Date`/`Time`/`Timestamp` even at runtime, and `uuid`, where pgjdbc's plain - * `getObject(int)` does return a `java.util.UUID` at runtime (`PgResultSet.internalGetObject` - * special-cases the Postgres `uuid` type by name) but the static type is still `Any` — the class hint - * is required in both cases, for different reasons (pgjdbc 42.7.13's - * `PgResultSet.getObject(int, Class)` special-cases each of these classes explicitly). - * `null` only for types read via a named, non-generic getter (e.g. `getString`, `getBlob`, `getBytes`), - * whose declared return type already is [kotlinType]. - * @property convertOffsetDateTimeToInstant When `true`, the wire value read via [getterClassHint] - * (always [OffsetDateTime] in this case) is converted with `.toInstant()` after reading, and a - * [kotlinType] ([Instant]) value is converted back with `OffsetDateTime.ofInstant(value, - * ZoneOffset.UTC)` before writing. Set only for `timestamptz`, whose wire representation - * ([OffsetDateTime]) differs from the Kotlin representation the non-domain scalar path uses - * ([Instant], see [InstantSqlMappable]) — every other type's wire and Kotlin representations are - * the same type, so this is `false` for them. - */ -internal data class JdbcTypeInfo( - val getterName: String, - val setterName: String, - val isPrimitive: Boolean, - val sqlTypeConstant: String, - val useSqlTypeHint: Boolean = false, - val kotlinType: TypeName, - val getterClassHint: ClassName? = null, - val convertOffsetDateTimeToInstant: Boolean = false, -) - /** * [SqlMappable] for a column that uses a `norm.ColumnAdapter` for encode/decode. * * Covers auto-generated adapters (enums, domains) and user-configured adapters. The adapter's - * wire type is described by [jdbcTypeInfo], which determines the JDBC getter/setter methods. + * wire type is described by [codec], which determines the JDBC getter/setter methods. * - * Generated types don't exist at generator time, so [klass] is not available — use [typeName] instead. + * Generated types don't exist at generator time, so there is no [kotlin.reflect.KClass] to expose + * — [typeName] is the only way to describe the type. * * The generated read/write code references an adapter property (e.g., `emailAdapter`) on the enclosing * `PostgresQueries` class, which is visible inside the `ResultSet`/`PreparedStatement` receiver lambdas @@ -382,141 +357,47 @@ internal data class JdbcTypeInfo( * or a parameterized type like `kotlin.collections.Map`). * @param adapterPropertyName The property name on `PostgresQueries` for the adapter (e.g., `"emailAdapter"`). * @param notNull Whether the column is `NOT NULL`. - * @param jdbcTypeInfo JDBC method info for the adapter's wire type. + * @param codec Wire-level access for the adapter's wire type. */ internal class AdaptedTypeSqlMappable( private val applicationTypeName: TypeName, private val adapterPropertyName: String, private val notNull: Boolean, - private val jdbcTypeInfo: JdbcTypeInfo, + private val codec: WireCodec, ) : SqlMappable { - override val klass: KClass<*> - get() = throw UnsupportedOperationException( - "Generated type $applicationTypeName has no KClass at generator time. Use typeName instead.", - ) - override val typeName: TypeName get() = applicationTypeName override val statementAction: (index: Int, parameterName: CodeBlock) -> CodeBlock get() = if (notNull) { - if (jdbcTypeInfo.useSqlTypeHint) { - { index, parameterName -> - CodeBlock.of( - "setObject(%L, %N.encode(%L), %T.%N)", - index, - adapterPropertyName, - parameterName, - Types::class, - jdbcTypeInfo.sqlTypeConstant, - ) - } - } else { - { index, parameterName -> - CodeBlock.of( - "%N(%L, %L)", - jdbcTypeInfo.setterName, - index, - encodedValueExpression(parameterName), - ) - } - } + { index, parameterName -> codec.write(index, encode(parameterName)) } } else { - if (jdbcTypeInfo.useSqlTypeHint) { - { index, parameterName -> - CodeBlock.of( - "%L?.let { setObject(%L, %N.encode(it), %T.%N) } ?: setNull(%L, %T.%N)", - parameterName, - index, - adapterPropertyName, - Types::class, - jdbcTypeInfo.sqlTypeConstant, - index, - Types::class, - jdbcTypeInfo.sqlTypeConstant, - ) - } - } else { - { index, parameterName -> - CodeBlock.of( - "%L?.let { %N(%L, %L) } ?: setNull(%L, %T.%N)", - parameterName, - jdbcTypeInfo.setterName, - index, - encodedValueExpression(CodeBlock.of("it")), - index, - Types::class, - jdbcTypeInfo.sqlTypeConstant, - ) - } + { index, parameterName -> + CodeBlock.of( + "%L?.let { %L } ?: %L", + parameterName, + codec.write(index, encode(CodeBlock.of("it"))), + codec.writeNull(index), + ) } } override val resultSetAction: (index: Int) -> CodeBlock get() = if (notNull) { - { index -> CodeBlock.of("%N.decode(%L)", adapterPropertyName, readExpression(index, nullable = false)) } - } else if (jdbcTypeInfo.isPrimitive) { - { index -> - CodeBlock.of( - "%L.takeUnless { wasNull() }?.let { %N.decode(it) }", - rawReadExpression(index), - adapterPropertyName, - ) - } + { index -> CodeBlock.of("%N.decode(%L)", adapterPropertyName, codec.read(index, false)) } } else { { index -> - CodeBlock.of( - "%L?.let { %N.decode(it) }", - readExpression(index, nullable = true), - adapterPropertyName, - ) + CodeBlock.of("%L?.let { %N.decode(it) }", codec.read(index, true), adapterPropertyName) } } /** * The encoded, wire-ready form of [valueExpression] (an already-non-null Kotlin value of - * [applicationTypeName]'s underlying domain/adapter base type): `adapter.encode(value)`, or — only - * when [JdbcTypeInfo.convertOffsetDateTimeToInstant] is set — that same encode call converted from - * [Instant] to [OffsetDateTime], since `timestamptz`'s wire representation is [OffsetDateTime] but - * its Kotlin representation ([kotlinType][JdbcTypeInfo.kotlinType]) is [Instant]. See - * [JdbcTypeInfo.convertOffsetDateTimeToInstant]'s KDoc. + * [applicationTypeName]'s underlying domain/adapter base type): `adapter.encode(value)`. */ - private fun encodedValueExpression(valueExpression: CodeBlock): CodeBlock { - val encoded = CodeBlock.of("%N.encode(%L)", adapterPropertyName, valueExpression) - return if (jdbcTypeInfo.convertOffsetDateTimeToInstant) { - CodeBlock.of("%T.ofInstant(%L, %T.UTC)", OffsetDateTime::class, encoded, ZoneOffset::class) - } else { - encoded - } - } - - /** - * The raw JDBC read at [index]: `getterName(index)`, or — when [JdbcTypeInfo.getterClassHint] is - * set — `getObject(index, X::class.java)`. See [JdbcTypeInfo.getterClassHint]'s KDoc for why some - * types need the class-qualified form. - */ - private fun rawReadExpression(index: Int): CodeBlock = if (jdbcTypeInfo.getterClassHint != null) { - CodeBlock.of("%N(%L, %T::class.java)", jdbcTypeInfo.getterName, index, jdbcTypeInfo.getterClassHint) - } else { - CodeBlock.of("%N(%L)", jdbcTypeInfo.getterName, index) - } - - /** - * [rawReadExpression] converted to [JdbcTypeInfo.kotlinType] — only [JdbcTypeInfo.convertOffsetDateTimeToInstant] - * types need a conversion, applied as a safe call (`?.toInstant()`) when [nullable] so a `NULL` - * column value stays `null` rather than throwing on the safe-call receiver. - */ - private fun readExpression(index: Int, nullable: Boolean): CodeBlock { - val raw = rawReadExpression(index) - return if (!jdbcTypeInfo.convertOffsetDateTimeToInstant) { - raw - } else if (nullable) { - CodeBlock.of("%L?.toInstant()", raw) - } else { - CodeBlock.of("%L.toInstant()", raw) - } - } + private fun encode(valueExpression: CodeBlock): CodeBlock = + CodeBlock.of("%N.encode(%L)", adapterPropertyName, valueExpression) } /** @@ -551,11 +432,6 @@ internal class AdaptedArrayTypeSqlMappable( private val decodeArrayMember = MemberName("norm", "decodeArray", isExtension = true) private val encodeToSqlArrayMember = MemberName("norm", "encodeToSqlArray", isExtension = true) - override val klass: KClass<*> - get() = throw UnsupportedOperationException( - "Generated array type Array<$applicationTypeName?> has no KClass at generator time. Use typeName instead.", - ) - override val typeName: TypeName get() = ARRAY.parameterizedBy(applicationTypeName.copy(nullable = true)) diff --git a/generator/src/main/kotlin/norm/generator/SqlOutputClause.kt b/generator/src/main/kotlin/norm/generator/SqlOutputClause.kt index 462f430e..b3fcd3e4 100644 --- a/generator/src/main/kotlin/norm/generator/SqlOutputClause.kt +++ b/generator/src/main/kotlin/norm/generator/SqlOutputClause.kt @@ -7,23 +7,18 @@ package norm.generator * @property columnName The column's logical name for a simple column reference — `null` for a * computed expression. For a quoted reference (`"My Col"`, `"He""llo"`), this is the identifier * PostgreSQL itself resolves to: surrounding quotes removed and any doubled `""` escape - * collapsed to the single literal `"` it represents (checked directly against a live - * PostgreSQL 18: `ResultSetMetaData.getColumnName` for `SELECT "He""llo" FROM (SELECT 1 AS - * "He""llo") s` reports `He"llo` — no quotes, escape already collapsed — which is exactly what - * this property must agree with, since [JdbcAnalyzer]'s own `originalName` falls back to - * `columnName` first and only reaches `getColumnName` when this is `null`). This is never the - * raw, quote-decorated source text — see [isColumnNameQuoted] for how the quoted/unquoted - * distinction PostgreSQL folding depends on is preserved instead, alongside the logical value - * rather than encoded inside it. + * collapsed to the single literal `"` it represents. On PostgreSQL 18, + * `ResultSetMetaData.getColumnName` for `SELECT "He""llo" FROM (SELECT 1 AS "He""llo") s` + * reports `He"llo` — no quotes, escape already collapsed — which this property must agree with. + * Never the raw, quote-decorated source text; see [isColumnNameQuoted] for how the + * quoted/unquoted distinction is preserved instead. * @property tableName The table qualifier's logical name for a qualified reference (e.g. `author` * in `author.name`, or `My Table` in `"My Table".name`) — same logical-value convention as * [columnName], and `null` for an unqualified reference or a computed expression. - * @property isColumnNameQuoted Whether [columnName] came from a quoted source identifier — - * `false` when [columnName] is `null`. PostgreSQL folds a quoted identifier reference - * exactly (case preserved, never lowercased) and an unquoted one to lowercase; since - * [columnName] itself no longer carries the quotes that would otherwise signal which rule - * applies (they were already removed to produce the logical value), this flag is what a caller - * doing that folding (see `foldIdentifier`'s two-argument overload) must consult instead. + * @property isColumnNameQuoted Whether [columnName] came from a quoted source identifier — `false` + * when [columnName] is `null`. [columnName] no longer carries the quotes that would otherwise + * signal whether PostgreSQL folds it (quoted: case preserved) or not (unquoted: folded to + * lowercase), so this flag is what a caller doing that folding must consult instead. * @property isTableNameQuoted Whether [tableName] came from a quoted source identifier — same * convention as [isColumnNameQuoted], `false` when [tableName] is `null`. */ @@ -38,15 +33,13 @@ internal data class SelectItem( /** * Parses the output clause of a SQL statement to extract individual items. * - * Paired positionally against `java.sql.ResultSetMetaData` columns by - * `JdbcAnalyzer.buildResultColumns`, so the search is restricted to the statement's main query — - * after any leading `WITH` clause's CTEs, via [parseCteClause]'s [ParsedCteClause.mainQueryStart] - * — and `SELECT` is located with [findTopLevelKeyword] (depth-0, lexically aware, never - * `String.indexOf`), so a nested `SELECT` or a keyword-like substring inside a literal/comment is - * never mistaken for the real clause. `RETURNING` is located with [findTopLevelReturningKeyword] - * instead — see its KDoc for why a plain [findTopLevelKeyword] search is not enough — and is only - * searched for when the window's own leading keyword is DML (`INSERT`/`UPDATE`/`DELETE`/`MERGE`), - * since `RETURNING` is not reserved and is otherwise legal as a plain `SELECT`'s column alias. + * Paired positionally against `java.sql.ResultSetMetaData` columns, so the search is restricted to + * the statement's main query — after any leading `WITH` clause's CTEs — and `SELECT` is located + * with [findTopLevelKeyword] (depth-0, lexically aware), so a nested `SELECT` or a keyword-like + * substring inside a literal/comment is never mistaken for the real clause. `RETURNING` is located + * with [findTopLevelReturningKeyword] instead, and is only searched for when the window's own + * leading keyword is DML (`INSERT`/`UPDATE`/`DELETE`/`MERGE`), since `RETURNING` is not reserved + * and is otherwise legal as a plain `SELECT`'s column alias. * * Handles: * - Simple columns: `title` → expression=`title`, columnName=`title` @@ -60,28 +53,23 @@ internal data class SelectItem( * [skipOptionalSetQuantifier] before the clause is split into items, so the first item's * `expression`/`columnName` reflect the bare column, not the quantifier glued onto it (`SELECT * DISTINCT x, id FROM t` → first item is `x`, not `DISTINCTx`). This must happen here, not inside - * [isStarItem]: [isStarItem] normalizes by stripping all whitespace, so `ALL 2.*a` and a genuine - * star on a table named `all2` (`SELECT all2.* a, p FROM all2` — valid, 3 columns) become - * textually identical (`ALL2.*a`/`all2.*a`) once whitespace is gone; only stripping the quantifier - * before that normalization, using the still-whitespace-intact `window`, can tell them apart — - * `skipOptionalKeyword`'s own word-boundary check is what refuses to match `ALL` as a prefix of the - * identifier `all2` (`SELECT ALL 2.*a lbl, b FROM t` is arithmetic, 2 columns, distinct - * from the `all2` table case despite normalizing identically further down the pipeline). + * [isStarItem]: [isStarItem]'s own whitespace-stripping turns `ALL 2.*a` into `ALL2.*a` and a + * genuine star on a table named `all2` into `all2.*a`, which differ only in the case of letters + * an unquoted identifier reference folds anyway; only stripping the quantifier first, while + * whitespace is still intact, tells them apart. `SELECT ALL 2.*a lbl, b FROM t` is valid + * arithmetic (2 columns), distinct from `SELECT all2.* a, p FROM all2` (3 columns, a real star on + * table `all2`). * - * A star item (`*`/`table.*`, via [isStarItem] — which also recognizes an implicit alias on a - * star, including a quoted, `U&`-escaped, or non-ASCII unquoted alias, though not claimed - * exhaustive — see [isStarItem]'s own KDoc) expands to however many columns the starred relation - * has, shifting every later item onto the wrong metadata column; an item before a star is + * A star item (`*`/`table.*`, via [isStarItem]) expands to however many columns the starred + * relation has, shifting every later item onto the wrong metadata column; an item before a star is * unaffected by that unknown width, so only the first star and everything after it is dropped — a * lone star is left alone, and items at/after a star fall back to metadata unresolved. * * @return Items strictly before the first star, if any; the full list if there is no star or it's * a single star item; or empty if the output clause can't be found (e.g. a `VALUES` list, a * `TABLE` shorthand, or two separately parenthesized set-operation branches like `(SELECT a) - * UNION (SELECT b)` — none have a depth-0 `SELECT`/`RETURNING`; a main query or CTE body wrapped - * in one redundant pair of parentheses, `(SELECT ...)`, does resolve — see - * [stripRedundantOuterParentheses]). Both consumers degrade safely for a missing item, falling - * back to `ResultSetMetaData.getColumnName` rather than reporting a wrong original name. + * UNION (SELECT b)`). Both consumers degrade safely for a missing item, falling back to + * `ResultSetMetaData.getColumnName` rather than reporting a wrong original name. */ internal fun parseSelectItems(sql: String): List = parseOutputItemsWithAlias(sql).map { it.selectItem } @@ -103,19 +91,13 @@ private fun mainQueryWindow(sql: String): String { /** * Whether [sql]'s main query ([mainQueryWindow]) contains a top-level `UNION`/`INTERSECT`/`EXCEPT` * keyword: this statement's own visible `SELECT`/`RETURNING` list is only one branch of a set - * operation, so [parseSelectItems] parsed only that branch's items, never the other branch(es)' own - * (possibly differently-computed) expressions for the same result column. - * - * [TypeRepository.buildTypeProjectionForQuery] uses this to suppress a computed expression's own - * `@property` source-reference line under a set operation — reporting `UPPER(x)` alone for `SELECT - * UPPER(x) AS u FROM t UNION SELECT LOWER(x) FROM t` would present one branch as the whole answer. A - * bare column reference is unaffected: PostgreSQL itself names a set operation's whole result column - * after branch 1 alone, so echoing that name back is not misleading. + * operation, so [parseSelectItems] parsed only that branch's items, never the other branch(es)' + * own (possibly differently-computed) expressions for the same result column. A bare column + * reference is unaffected: PostgreSQL itself names a set operation's whole result column after + * branch 1 alone. * * All three keywords are PostgreSQL reserved words, so an unquoted occurrence can never be a - * column/table identifier or alias — only [findTopLevelKeyword]'s lexical- and depth-awareness is - * needed, not the additional alias-position gating [findTopLevelReturningKeyword] needs for the - * non-reserved `RETURNING`. + * column/table identifier or alias. */ internal fun hasTopLevelSetOperation(sql: String): Boolean { val window = mainQueryWindow(sql) @@ -142,18 +124,15 @@ private val SET_OPERATION_KEYWORDS = listOf("UNION", "INTERSECT", "EXCEPT") internal data class OutputItemWithAlias(val selectItem: SelectItem, val alias: String?) /** - * The shared parsing core behind both [parseSelectItems] (which discards [OutputItemWithAlias.alias]) - * and [resolveNodeTreeProvenanceExpression] (which needs the alias to cross-validate a CTE body - * item's own name against the node tree's authoritative `:resname` — see that function's KDoc for - * why the alias can't be dropped there). Locates the same output clause [parseSelectItems] - * documents finding — see its KDoc for the window/`RETURNING`-gating/star-truncation rules, all of - * which apply identically here. The window itself is [mainQueryWindow] — see its KDoc for why - * [hasTopLevelSetOperation] must compute that same window rather than its own copy. + * The shared parsing core behind both [parseSelectItems] and [resolveNodeTreeProvenanceExpression] + * (which needs [OutputItemWithAlias.alias] to cross-validate a CTE body item's own name against + * the node tree's authoritative `:resname`). Locates the same output clause [parseSelectItems] + * documents finding — the window/`RETURNING`-gating/star-truncation rules there apply identically + * here. */ internal fun parseOutputItemsWithAlias(sql: String): List { val window = mainQueryWindow(sql) - // See parseSelectItems' KDoc for why RETURNING is gated on the main query's own leading keyword. val leadingKeywordStart = skipWhitespaceAndComments(window, 0) val isDmlMainQuery = listOf("INSERT", "UPDATE", "DELETE", "MERGE").any { keyword -> skipOptionalKeyword(window, leadingKeywordStart, keyword) != leadingKeywordStart @@ -168,11 +147,8 @@ internal fun parseOutputItemsWithAlias(sql: String): List { if (returningIndex >= 0) { afterKeyword = returningIndex + "RETURNING".length // PostgreSQL 18's `RETURNING WITH (OLD AS o, NEW AS n) o.x, n.x` prologue is not part of the - // first item's own expression — skip past it via parseOldNewAliasPrologue so itemsStart lands - // on the real first item, not on `WITH (OLD AS o, NEW AS n) o.x` (which parseColumnReference - // cannot make sense of, so it would otherwise be embedded verbatim in generated KDoc as that - // column's expression). - itemsStart = parseOldNewAliasPrologue(window, afterKeyword).second + // first item's own expression; skip past it so itemsStart lands on the real first item. + itemsStart = parseOldNewAliasPrologue(window, afterKeyword) // RETURNING clauses are terminal — no FROM keyword follows hasFromClause = false } else if (selectIndex >= 0) { @@ -229,21 +205,14 @@ private fun stripRedundantOuterParentheses(text: String): String { } /** - * Skips an optional leading SQL set quantifier starting at or after [position] in [sql]: `ALL`, - * or `DISTINCT` optionally followed by `ON (` ... `)`. Used by [parseSelectItems] to separate a - * `SELECT`'s quantifier from its first real item before [isStarItem] ever sees it — see that - * function's KDoc for why this can't be done inside [isStarItem] itself. - * - * [position] is the index right after the `SELECT` keyword — still followed by whitespace, since - * [skipOptionalKeyword] (unlike [skipWhitespaceAndComments]) requires its keyword to start exactly - * where it's told to look, with no leading separator of its own to skip. This function skips that - * whitespace/comments first, then tries `ALL`/`DISTINCT` at the resulting position — otherwise - * `regionMatches` would fail immediately on the space between `SELECT` and the quantifier, this - * function would report no quantifier present, and `parseSelectItems` would go right back to - * feeding [isStarItem] the fused, whitespace-bearing text this function exists to prevent. + * Skips an optional leading SQL set quantifier starting at or after [position] in [sql]: `ALL`, or + * `DISTINCT` optionally followed by `ON (` ... `)`. * - * [skipOptionalKeyword]'s own word-boundary check is what keeps `ALL`/`DISTINCT` from matching a - * longer identifier that merely starts with those letters (`all2`, `distinctive_column`). + * [position] is the index right after the `SELECT` keyword, still followed by whitespace: this + * function skips that whitespace/comments first, then tries `ALL`/`DISTINCT` at the resulting + * position. [skipOptionalKeyword]'s own word-boundary check is what keeps `ALL`/`DISTINCT` from + * matching a longer identifier that merely starts with those letters (`all2`, + * `distinctive_column`). * * @return The index immediately after the quantifier (and any trailing whitespace/comments), or * [position] unchanged if there is no quantifier there. @@ -270,57 +239,29 @@ private fun skipOptionalSetQuantifier(sql: String, position: Int): Int { * Splits a select item into its expression and alias parts. * * Handles `expression AS alias` patterns, respecting parentheses so that - * `CAST(x AS text) AS my_col` correctly identifies `my_col` as the alias. - * - * Skips string literals, quoted identifiers, dollar-quoted strings, and comments via - * [skipLexicalToken], so an `AS`-like substring or an unbalanced paren inside one of those is - * not mistaken for a real `AS` keyword or a real parenthesis. + * `CAST(x AS text) AS my_col` correctly identifies `my_col` as the alias. Skips string literals, + * quoted identifiers, dollar-quoted strings, and comments via [skipLexicalToken], so an `AS`-like + * substring or an unbalanced paren inside one of those is not mistaken for a real `AS` keyword or + * a real parenthesis. * - * Tracks `(`/`)` only, deliberately not `[`/`]` — unlike [splitAtTopLevel], which shares one depth - * counter across both bracket kinds because it scans raw, not-yet-split clause text, where a - * top-level-looking `,` can sit directly inside an unsplit `ARRAY[...]` literal (see - * [splitAtTopLevel]'s own KDoc). [extractAlias] instead only ever - * receives an item [parseOutputItemsWithAlias] already split via [splitAtTopLevel] — so any - * `[`/`]` pair the item contains is, by construction of that prior split, already self-balanced - * and cannot itself hold an unmatched `(`/`)` inside it either. Tracking `[`/`]` here would - * therefore never change which `AS` this scan finds: PostgreSQL syntax also never lets a bare `AS` - * appear directly inside `[...]` (an array literal holds element expressions, not aliases) without - * that `AS` also sitting inside some `(...)` this scan already tracks (`ARRAY[CAST(x AS int)]`, for - * instance, has its `AS` inside `CAST(...)`'s own parentheses). + * Tracks `(`/`)` only, not `[`/`]`: [extractAlias] only ever receives an item already split at the + * top level ([splitAtTopLevel]), so any `[`/`]` pair it contains is already self-balanced and + * cannot itself hold an unmatched `(`/`)`. * - * The word-boundary check on either side of a candidate `AS`/`as` uses [isIdentifierChar] — the - * same predicate [findTopLevelKeyword] and every other keyword scanner in this file use — rather + * The word-boundary check on either side of a candidate `AS`/`as` uses [isIdentifierChar] rather * than `Char.isWhitespace()`: PostgreSQL's `AS` keyword only needs to not be fused into a longer - * identifier on either side, not to be surrounded by literal whitespace. On - * PostgreSQL 18.4: `SELECT (1)AS b` returns column `b` — `AS` directly abuts the closing `)` with - * no whitespace, and `)` is not an identifier character, so this is the real keyword. Conversely - * `SELECT 1 AS$b` returns column `as$b`, a single implicit alias identifier — `$` is a valid - * identifier-continuation character (see [isIdentifierChar]), so `AS$b` is one word, not the - * keyword `AS` followed by `$b`. - * - * This affects `columnName` whenever a glued alias would otherwise prevent the pre-`AS` text from - * matching [COLUMN_REFERENCE] on its own, but the split-off expression matches it once separated. - * `columnName` is derived by [parseOutputItemsWithAlias] from the split `expression` via - * [parseColumnReference] — [parseOutputItemsWithAlias] is this function's only caller, and does - * `val (expression, alias) = extractAlias(item)`, keeping the alias half (unlike [parseSelectItems], - * which discards it) — so whatever the split changes `expression` to feeds directly into that - * derivation. On PostgreSQL 18.4: `SELECT a AS"b", id FROM t` is valid (columns `b`, - * `id`; PostgreSQL reports `a` as the source column of the first result column) — `AS"b"` (no - * space before the quote) is recognized as the keyword here, since `"` is not an identifier - * character, so the right-hand boundary holds without requiring whitespace; the item splits into - * `expression="a"`, which matches [COLUMN_REFERENCE], giving `columnName="a"`, the name PostgreSQL - * itself reports. Not every glued case benefits this way: a glued expression like `(age)AS b` - * still fails to match [COLUMN_REFERENCE] once split, because the split-off `expression` (`(age)`) - * contains parentheses regardless of the split, so `columnName` stays `null` for that shape either - * way. The effect on `expression` itself is unconditional, independent of `columnName`: - * `TypeRepository.buildTypeProjectionForQuery` embeds `expression` verbatim in generated KDoc for - * a computed expression (`selectItem.columnName == null && column.table == null`). + * identifier on either side, not to be surrounded by literal whitespace. On PostgreSQL 18.4, + * `SELECT (1)AS b` returns column `b` — `AS` directly abuts the closing `)` with no whitespace, + * and `)` is not an identifier character, so this is the real keyword. Likewise `SELECT a AS"b", + * id FROM t` (columns `b`, `id`, with `a` as the first column's source) recognizes `AS"b"` as the + * keyword since `"` is not an identifier character either. Conversely `SELECT 1 AS$b` returns + * column `as$b`, a single implicit alias identifier — `$` is an identifier-continuation character, + * so `AS$b` is one word, not the keyword `AS` followed by `$b`. * * @return A pair of (expression, alias). `alias` is `null` when there is no `AS` keyword at all, * and when there is one but [parseAliasToken] finds nothing that legitimately looks like an - * alias right after it (see that function's own KDoc — a trailing comment, an unterminated - * quote, or a string literal where an alias should be, none of which contribute a real alias - * name). + * alias right after it: a trailing comment, an unterminated quote, or a string literal where an + * alias should be, none of which contribute a real alias name. */ private fun extractAlias(item: String): Pair { // Find the last top-level AS keyword @@ -337,12 +278,10 @@ private fun extractAlias(item: String): Pair { '(' -> depth++ ')' -> { depth-- - // A bare ')' with no matching '(' means [item] is not the well-formed, already-top-level - // expression this scan assumes — see findTopLevelKeyword's own KDoc on why bailing (rather - // than clamping depth at 0 and continuing) is the safe direction: an unbalanced scan's - // assumptions are already void, so a loud "no alias found" is safer than a depth count that - // silently recovers and may misplace a later real AS keyword. [item] is returned unsplit, - // the same fallback [extractAlias] already uses when no top-level AS exists at all. + // A bare ')' with no matching '(' means [item] isn't the well-formed, already-top-level + // expression this scan assumes. Bail rather than clamp depth at 0 and continue: returning + // [item] unsplit is safer than a depth count that silently recovers and may misplace a + // later real AS keyword. if (depth < 0) return item to null } 'A', 'a' -> if (depth == 0 && i + 1 < item.length && (item[i + 1] == 'S' || item[i + 1] == 's')) { @@ -364,23 +303,22 @@ private fun extractAlias(item: String): Pair { } /** - * Extracts exactly the alias token starting at or after [start] in [item] (the position right after - * the `AS` keyword [extractAlias] already found) — a bare identifier or a double-quoted one, + * Extracts exactly the alias token starting at or after [start] in [item] (the position right + * after the `AS` keyword [extractAlias] already found) — a bare identifier or a double-quoted one, * discarding any leading/trailing whitespace or comments around it (`AS /* c */ ux -- note`). * - * The quoted branch is escape-aware, via the same [QUOTED_IDENTIFIER_PATTERN] [COLUMN_REFERENCE] - * uses, so a doubled `""` inside the alias (`AS "zz""q"`) is treated as an escaped literal `"`, not - * the token's end — an escape-unaware scan would stop at the first `"`, returning the truncated - * fragment `"zz`, which still looks quoted and folds to the shorter name `zz`, wrongly colliding + * The quoted branch is escape-aware: a doubled `""` inside the alias (`AS "zz""q"`) is treated as + * an escaped literal `"`, not the token's end. An escape-unaware scan would stop at the first `"`, + * returning the truncated fragment `"zz`, which folds to the shorter name `zz`, wrongly colliding * with an unrelated `zz` elsewhere in the same body. * - * @return The alias token — with its surrounding quotes and any internal `""` escape still attached - * when quoted, exactly as [foldIdentifier] expects — or `null` when there is no legitimate alias - * token: nothing but whitespace/comments to the end of [item], an unterminated quoted identifier, a - * character that can neither start a bare identifier nor open a quoted one (e.g. `AS 'x'`, not - * legal PostgreSQL), or anything other than trailing whitespace/comments following the token - * (`AS ux zz` is not legal PostgreSQL either, so this returns `null` rather than silently - * discarding `zz`). + * @return The alias token — with its surrounding quotes and any internal `""` escape still + * attached when quoted — or `null` when there is no legitimate alias token: nothing but + * whitespace/comments to the end of [item], an unterminated quoted identifier, a character that + * can neither start a bare identifier nor open a quoted one (e.g. `AS 'x'`, not legal + * PostgreSQL), or anything other than trailing whitespace/comments following the token (`AS ux + * zz` is not legal PostgreSQL either, so this returns `null` rather than silently discarding + * `zz`). */ private fun parseAliasToken(item: String, start: Int): String? { val tokenStart = skipWhitespaceAndComments(item, start) @@ -413,25 +351,16 @@ private fun parseAliasToken(item: String, start: Int): String? { * computed expression. * * A quoted position whose logical value comes out empty (`SELECT "" FROM t`) is treated as no - * match at all — the same `columnName = null`/`tableName = null` fallback as an expression that - * doesn't match [COLUMN_REFERENCE] to begin with — rather than an empty-string name: PostgreSQL - * itself rejects a zero-length delimited identifier outright (`zero-length delimited identifier` - * is a real syntax error), so this shape can never actually reach here from a query PostgreSQL - * accepted, but an empty non-null name is a worse `null` than `null` itself — it would make - * `JdbcAnalyzer.buildResultColumns`' `originalName` fall to `""` instead of correctly falling back - * to `ResultSetMetaData.getColumnName`, exactly the wrong-value-over-no-value mistake this whole - * file exists to avoid. + * match at all (`columnName = null`/`tableName = null`), never an empty-string name: PostgreSQL + * itself rejects a zero-length delimited identifier outright (`SELECT "" FROM t` is a syntax + * error), so this shape can never actually reach here from an accepted query, but an empty + * non-null name would still be a worse `null` than `null` itself. * - * An unquoted column/table name is folded via [foldAsciiCase] — PostgreSQL's own `downcase_identifier` - * behavior, ASCII `A`-`Z` only, not Kotlin's `String.lowercase()` — so [SelectItem.columnName]/ + * An unquoted column/table name is folded via [foldAsciiCase] — PostgreSQL's own + * `downcase_identifier` behavior, ASCII `A`-`Z` only — so [SelectItem.columnName]/ * [SelectItem.tableName] agree with what `ResultSetMetaData.getColumnName` reports for the same - * reference (an unfolded `ID` would miss `JdbcAnalyzer.buildResultColumns`'s `catalog.findColumn` - * lookup and drop the column's Postgres comment). A quoted name is never folded — quoting is how - * PostgreSQL preserves a name's original case against this default folding. - * - * Also called directly by [resolveNodeTreeProvenanceExpression] to re-classify an already - * alias-stripped CTE body item, since the bare-column check needs a fresh classification of the - * stripped text rather than the item's own pre-strip [SelectItem.columnName]. + * reference. A quoted name is never folded — quoting is how PostgreSQL preserves a name's + * original case against this default folding. */ internal fun parseColumnReference(expression: String): SelectItem { val trimmed = expression.trim() @@ -462,52 +391,22 @@ internal fun parseColumnReference(expression: String): SelectItem { } /** - * Parses PostgreSQL 18's optional `RETURNING WITH (OLD AS alias, NEW AS alias) ...` prologue, + * Skips PostgreSQL 18's optional `RETURNING WITH (OLD AS alias, NEW AS alias) ...` prologue, * which declares a custom name for referring to the `OLD`/`NEW` pseudo-relations in the * `RETURNING` list that follows (e.g. `RETURNING WITH (OLD AS o, NEW AS n) o.name, n.name`). * * @param dml The data-modifying statement. * @param afterReturningKeyword The index in [dml] immediately after the `RETURNING` keyword. - * @return The declared alias names (empty if there is no prologue) paired with the index where - * the actual `RETURNING` item list begins — after the prologue's closing `)`, or unchanged if - * there is none. + * @return The index where the actual `RETURNING` item list begins — after the prologue's closing + * `)`, or unchanged if there is none. */ -private fun parseOldNewAliasPrologue(dml: String, afterReturningKeyword: Int): Pair, Int> { +private fun parseOldNewAliasPrologue(dml: String, afterReturningKeyword: Int): Int { val beforeWith = skipWhitespaceAndComments(dml, afterReturningKeyword) val afterWith = skipOptionalKeyword(dml, beforeWith, "WITH") if (afterWith == beforeWith || afterWith >= dml.length || dml[afterWith] != '(') { - return emptySet() to afterReturningKeyword + return afterReturningKeyword } val closeParenthesis = findMatchingCloseParenthesis(dml, afterWith) - if (closeParenthesis < 0) return emptySet() to afterReturningKeyword - val aliasNames = splitAtTopLevel(dml.substring(afterWith + 1, closeParenthesis), ',') - .mapNotNull { entry -> parseOldNewAliasName(entry) } - .toSet() - return aliasNames to (closeParenthesis + 1) -} - -/** - * Extracts the alias name declared after `AS` in one comma-separated entry of a - * `RETURNING WITH (OLD AS o, NEW AS n)` prologue (e.g. the `o` in `OLD AS o`), or `null` if - * [entry] has no top-level `AS`. - * - * Whitespace and comments between `AS` and the alias, and trailing the alias before the entry - * ends, are skipped rather than captured — PostgreSQL accepts both (`OLD AS o /*c*/`, on - * PostgreSQL 18) and neither is part of the declared name. A naive - * `substring(asIndex + 2).trim()` captured a trailing comment as part of the alias (`o /*c*/` - * instead of `o`), which then never matched any real `RETURNING` item reference and silently - * dropped the alias from the set of recognized `OLD`/`NEW` aliases. - */ -private fun parseOldNewAliasName(entry: String): String? { - val asIndex = findTopLevelKeyword(entry, "AS") - if (asIndex < 0) return null - val afterAs = skipWhitespaceAndComments(entry, asIndex + "AS".length) - if (afterAs >= entry.length) return null - if (entry[afterAs] == '"') { - val afterQuote = skipDoubleQuotedIdentifier(entry, afterAs) - return entry.substring(afterAs + 1, (afterQuote - 1).coerceAtLeast(afterAs + 1)) - } - var end = afterAs - while (end < entry.length && isIdentifierChar(entry[end])) end++ - return entry.substring(afterAs, end) + if (closeParenthesis < 0) return afterReturningKeyword + return closeParenthesis + 1 } diff --git a/generator/src/main/kotlin/norm/generator/SqlParameterInferrer.kt b/generator/src/main/kotlin/norm/generator/SqlParameterInferrer.kt index 5a84fe11..d2455b60 100644 --- a/generator/src/main/kotlin/norm/generator/SqlParameterInferrer.kt +++ b/generator/src/main/kotlin/norm/generator/SqlParameterInferrer.kt @@ -364,14 +364,10 @@ internal class SqlParameterInferrer(private val functionOverloads: Map { - result.append('\'') - i++ - while (i < sql.length) { - if (sql[i] == '\'') { - result.append('\'') - i++ - if (i < sql.length && sql[i] == '\'') { - result.append('\'') - i++ - } else { - break - } - } else { - result.append(sql[i]) - i++ - } - } - } - sql[i] == '-' && i + 1 < sql.length && sql[i + 1] == '-' -> { - val eol = sql.indexOf('\n', i) - if (eol < 0) { - result.append(sql, i, sql.length) - i = sql.length - } else { - result.append(sql, i, eol + 1) - i = eol + 1 - } - } - sql[i] == '/' && i + 1 < sql.length && sql[i + 1] == '*' -> { - val close = sql.indexOf("*/", i + 2) - if (close < 0) { - result.append(sql, i, sql.length) - i = sql.length - } else { - result.append(sql, i, close + 2) - i = close + 2 - } - } - sql[i] == '?' -> { - result.append("NULL") - i++ - } - else -> { - result.append(sql[i]) - i++ - } - } - } - return result.toString() -} - -/** - * Replaces `?` parameter placeholders with typed non-null sentinel values. - * - * Each `?` is replaced with the corresponding sentinel from [sentinels] (consumed in order). - * If there are more `?` placeholders than sentinels, excess `?` are replaced with `NULL` - * (safe fallback). Question marks inside string literals and comments are left untouched. + * A `?` inside a string literal, `E''` escape string, quoted identifier, dollar-quoted string, line + * comment, or block comment is left alone. * * @param sql The SQL text with `?` parameter placeholders. - * @param sentinels Non-null sentinel expressions in parameter order (e.g., `"0::int4"`, `"''::text"`). - * @return The SQL with `?` replaced by sentinels. + * @param replacement Given a placeholder's 0-based parameter index, returns the SQL text to substitute for it. + * @return The SQL with each `?` replaced by [replacement]'s result for it. */ -internal fun replaceParameterPlaceholdersWithSentinels(sql: String, sentinels: List): String { +internal fun replaceParameterPlaceholders(sql: String, replacement: (parameterIndex: Int) -> String): String { if ('?' !in sql) return sql - val result = StringBuilder(sql.length + sentinels.sumOf { it.length }) - var characterIndex = 0 - var sentinelIndex = 0 - while (characterIndex < sql.length) { - when { - sql[characterIndex] == '\'' -> { - result.append('\'') - characterIndex++ - while (characterIndex < sql.length) { - if (sql[characterIndex] == '\'') { - result.append('\'') - characterIndex++ - if (characterIndex < sql.length && sql[characterIndex] == '\'') { - result.append('\'') - characterIndex++ - } else { - break - } - } else { - result.append(sql[characterIndex]) - characterIndex++ - } - } - } - sql[characterIndex] == '-' && characterIndex + 1 < sql.length && sql[characterIndex + 1] == '-' -> { - val endOfLine = sql.indexOf('\n', characterIndex) - if (endOfLine < 0) { - result.append(sql, characterIndex, sql.length) - characterIndex = sql.length - } else { - result.append(sql, characterIndex, endOfLine + 1) - characterIndex = endOfLine + 1 - } - } - sql[characterIndex] == '/' && characterIndex + 1 < sql.length && sql[characterIndex + 1] == '*' -> { - val close = sql.indexOf("*/", characterIndex + 2) - if (close < 0) { - result.append(sql, characterIndex, sql.length) - characterIndex = sql.length - } else { - result.append(sql, characterIndex, close + 2) - characterIndex = close + 2 - } - } - sql[characterIndex] == '?' -> { - result.append(sentinels.getOrElse(sentinelIndex) { "NULL" }) - sentinelIndex++ - characterIndex++ - } - else -> { - result.append(sql[characterIndex]) - characterIndex++ - } + val result = StringBuilder(sql.length + 16) + var index = 0 + var parameterIndex = 0 + while (index < sql.length) { + val afterToken = skipLexicalToken(sql, index) + if (afterToken != index) { + result.append(sql, index, afterToken) + index = afterToken + continue } + if (sql[index] == '?') result.append(replacement(parameterIndex++)) else result.append(sql[index]) + index++ } return result.toString() } diff --git a/generator/src/main/kotlin/norm/generator/SqlStarItem.kt b/generator/src/main/kotlin/norm/generator/SqlStarItem.kt index 74328491..0b3e38c8 100644 --- a/generator/src/main/kotlin/norm/generator/SqlStarItem.kt +++ b/generator/src/main/kotlin/norm/generator/SqlStarItem.kt @@ -2,32 +2,16 @@ package norm.generator /** * Removes every comment and all whitespace from [text], keeping every other character — including - * the full contents of a string literal, quoted identifier, or dollar-quoted string — verbatim - * and in relative order. A comment is always removed outright, with nothing put in its place. - * Comments are recognized (and dropped) via the same [skipLineComment]/[skipBlockComment] logic - * [skipWhitespaceAndComments] uses, applied at every position in [text] rather than only at an - * edge, so this finds and removes a comment anywhere — including one sitting between two - * otherwise-adjacent tokens (`tgt./*c*/ *`) — not merely one that leads or trails the whole - * string. A string literal, quoted identifier, or dollar-quoted string is recognized via - * [skipLexicalToken] and copied through unchanged (including any whitespace or `--`/`/* */`-shaped - * text inside it, which is real content, not a comment) rather than having its own contents - * stripped. + * the full contents of a string literal, quoted identifier, or dollar-quoted string — verbatim and + * in relative order. A comment is removed outright, wherever it sits, including one between two + * otherwise-adjacent tokens (`tgt./*c*/ *`), not merely one that leads or trails the whole string. * - * Used by [isStarItem] to normalize an item before its star check, which needs neither comments - * nor whitespace to survive — it locates a trailing implicit alias (via - * [findTrailingImplicitAliasStart]) and inspects the qualifier and star around it, so the - * separator that used to sit between them (whitespace, a comment, or nothing at all) makes no - * difference once it's gone. - * - * The result is a [StrippedText], not a plain `String`: deleting a separator PostgreSQL itself - * lexed on can fuse two characters that were never adjacent in the original query into a token - * that never existed — `1 - -1` (two independently-lexed `-` tokens) strips to `1--1`, which a - * naive re-lex of the output `String` alone would read as a `--` line comment. [StrippedText] - * carries, alongside the stripped characters, each one's original offset, so any later - * multi-character adjacency decision made against this output (a `--` line-comment or `/* */` - * block-comment opener, the `$` dollar-quote identifier lookback, the standalone-`E` escape-string - * lookback, a `''`/`""` doubled-quote escape) can be gated on whether the two characters were - * really adjacent in the query PostgreSQL itself lexed, not merely in this function's output. + * Returns a [StrippedText], not a plain `String`, because deleting a separator PostgreSQL itself + * lexed on can fuse two characters that were never adjacent into a token that never existed: + * `1 - -1` (two independently-lexed `-` tokens) strips to `1--1`, which a naive re-lex of the + * output `String` alone would read as a `--` line comment. [StrippedText] carries each stripped + * character's original offset so a later adjacency decision can be gated on whether two characters + * were really adjacent in the source text. */ internal fun stripCommentsAndWhitespace(text: String): StrippedText { val builder = StringBuilder(text.length) @@ -56,23 +40,16 @@ internal fun stripCommentsAndWhitespace(text: String): StrippedText { } /** - * The output of [stripCommentsAndWhitespace] — the stripped characters, plus each one's original + * The output of [stripCommentsAndWhitespace]: the stripped characters, plus each one's original * offset in the pre-stripping text, so [wereAdjacent] can answer whether two stripped characters - * that now sit next to each other in [text] genuinely were adjacent before stripping, or whether a - * comment/whitespace separator PostgreSQL itself lexed on used to sit between them. When a lexical - * token (a string literal, a quoted identifier, a dollar-quoted string) is copied through wholesale - * by [stripCommentsAndWhitespace], each of its characters maps to consecutive original offsets, so - * [wereAdjacent] is `true` throughout the token's own interior — only a genuinely removed - * whitespace/comment separator between two different tokens (or bare characters) ever breaks that - * consecutiveness. + * that now sit next to each other in [text] were genuinely adjacent before stripping, or had a + * whitespace/comment separator between them. * - * The raw stripped `String` is kept private: every lexer entry point stripped-path code needs + * The raw stripped `String` is kept private: every lexer function stripped-path code needs * ([skipLexicalToken], [findMatchingCloseParenthesis]) is exposed as a member here that threads - * `this` as the [OriginalAdjacency], so stripped-path code cannot reach the `String`-taking lexer - * functions and silently pass the wrong (or no) adjacency — the only way out to a plain `String` is - * [asPlainString], a single, deliberately named, greppable escape hatch for a caller (currently only - * [isStarQualifierAcceptable]) that does no lexing at all and therefore has no adjacency decision to - * gate. + * `this` as the [OriginalAdjacency], so stripped-path code cannot bypass the adjacency gate. + * [asPlainString] is the one escape hatch, for a caller that does no lexing and so has no + * adjacency decision to make. */ internal class StrippedText(private val text: String, private val originalOffsets: IntArray) : OriginalAdjacency { @@ -96,90 +73,47 @@ internal class StrippedText(private val text: String, private val originalOffset fun slice(from: Int, until: Int): StrippedText = StrippedText(text.substring(from, until), originalOffsets.copyOfRange(from, until)) - /** - * The original (pre-stripping) index that stripped index [strippedIndex] came from — used by - * [splitTrailingImplicitAlias] to translate a boundary located in stripped space (comments and - * whitespace already removed) back into the original text, so the expression half of the split - * can be sliced out of the caller's own, un-stripped item text, comments and all, rather than the - * stripped copy this class holds privately. - */ + /** The original (pre-stripping) index that stripped index [strippedIndex] came from. */ fun originalIndexOf(strippedIndex: Int): Int = originalOffsets[strippedIndex] - /** See [skipLexicalToken]'s KDoc — this threads `this` as the [OriginalAdjacency]. */ fun skipLexicalToken(position: Int): Int = norm.generator.skipLexicalToken(text, position, this) - /** See [findMatchingCloseParenthesis]'s KDoc — this threads `this` as the [OriginalAdjacency]. */ fun findMatchingCloseParenthesis(openParenthesisIndex: Int): Int = norm.generator.findMatchingCloseParenthesis(text, openParenthesisIndex, this) /** - * The single, deliberately named escape hatch out of this class's own lexer entry points, back - * to a plain `String` — see this class's own KDoc for why every other accessor exists instead of - * this one. Safe only for a caller that does no lexing at all, i.e. has no adjacency decision to - * gate; [isStarQualifierAcceptable] (inspecting the character class of a qualifier's trailing - * run) and [splitTrailingImplicitAlias] (extracting an already-located trailing alias segment's - * own text, which needs no further lexical stepping once found) are the only current callers. + * The escape hatch out of this class's lexer entry points, back to a plain `String`. Safe only + * for a caller that does no lexing and so has no adjacency decision to gate. */ fun asPlainString(): String = text } /** - * Check for whether a single `RETURNING`/`SELECT` item is a star (`*`, `tbl.*`), with or without - * an implicit (no-`AS`) alias — including parenthesized (`(tgt.*)`), and any placement of - * comments and whitespace around or between its tokens (`tgt.* /*c*/`, `tgt.* -- comment`, - * `tgt . *`, `tgt./*c*/ *`, `tgt.*whatever`, `tgt.*`/*c*/`whatever`), in any order relative to a - * wrapping `(...)` (a trailing comment can sit inside or outside the parentheses: `(tgt.*) -- c`, - * `(tgt.*) /*c*/`). - * - * Normalizes by first stripping every comment and all whitespace from the entire item — via - * [stripCommentsAndWhitespace], which finds a comment anywhere in the text, not merely at an - * edge. + * Whether a single `RETURNING`/`SELECT` item [item] is a star (`*`, `tbl.*`), with or without an + * implicit (no-`AS`) alias — including parenthesized (`(tgt.*)`) and with any placement of + * comments and whitespace around or between its tokens (`tgt.* /*c*/`, `tgt . *`, `tgt./*c*/ *`, + * `tgt.*whatever`). * * Two paths, tried in order: * - * Path 1 — the `text == "*" || text.endsWith(".*")` check (via [unwrapWrappingParentheses] then a - * literal suffix comparison). For [isStarItem], `false` is the dangerous answer (an unrecognized - * star lets a later item survive at its raw list position, silently shifted onto the wrong - * `ResultSetMetaData` column) and `true` is the safe one (later items are dropped, falling back to - * metadata names instead of a wrong mapping), so a rule that already answers `true` for some text - * must never be replaced by one that answers `false` for that same text — only new `true` answers - * may be added on top, never removed. This path alone already covers every shape whose normalized - * text ends in `.*` verbatim with nothing after it: `t.*`, a parenthesized composite expansion - * (`(t).*`, `(u.*)`, `((t.*))` — valid PostgreSQL syntax; the unwrap loop only requires - * the outermost wrapping pair to match, so it repeats until no more wrapping parens remain), a - * `DISTINCT ON (...)` prefix ([parseSelectItems] strips this before [isStarItem] ever sees it — - * see its KDoc), and a Unicode-escape quoted identifier (`U&"my*table".*`, also valid). + * Path 1 matches every shape whose text (comments and whitespace stripped, wrapping parentheses + * removed) ends in `.*` verbatim: `t.*`, a parenthesized composite expansion (`(t).*`, `(u.*)`, + * `((t.*))`), and a Unicode-escape identifier (`U&"my*table".*`). * - * Path 2 — reached only when path 1 answers `false`, i.e. only for an item with a trailing - * implicit alias (something other than `*` is the last character, so path 1's literal suffix check - * can never match). [findTrailingImplicitAliasStart] finds where that alias starts by walking the - * text forward and tracking the last segment seen — PostgreSQL's grammar guarantees an implicit - * alias is always the final token of a select item, so anchoring on "the last segment reaches - * exactly the end of the text" is grammar-backed, unlike enumerating everything that may - * legitimately precede a star's qualifying dot (parentheses, brackets, quotes, a `DISTINCT ON` - * prefix, a Unicode-escape identifier...), which is fragile against a qualifier shape not yet on - * the list. With the alias located, [unwrapWrappingParentheses] is applied to the prefix (the text - * with that alias - * removed), and the same path-1 logic is re-run on it: accept if the unwrapped prefix is `*`, or if - * it ends in `.*` and [isStarQualifierAcceptable] accepts the qualifier (everything before that - * final `.`) — the only additional check path 2 needs beyond path 1's, since a digit-leading run - * immediately before the dot (`2.` in `SELECT 2.*3 lbl, a FROM t` — arithmetic returning 2 - * columns, not a star) is the one lexical ambiguity a numeric literal creates with a real - * qualifying dot; see [isStarQualifierAcceptable]'s KDoc for why every other qualifier shape is - * accepted rather than enumerated. + * Path 2 is reached when path 1 answers `false`, for an item with a trailing implicit alias. + * [findTrailingImplicitAliasStart] locates where the alias starts; with it removed, path 1's check + * is re-run on the remaining prefix, additionally requiring [isStarQualifierAcceptable] to accept + * the qualifier when the prefix ends in `.*` — needed because a digit run immediately before the + * dot (`2.` in `SELECT 2.*3 lbl, a FROM t`, which returns 2 columns: arithmetic, not a star) is the + * one shape a real qualifying dot can be confused with. * - * On PostgreSQL 18.4: `SELECT u.*whatever, preferences FROM users u` and `SELECT - * u.*`/*c*/`whatever, preferences FROM users u` both return 5 columns (star expands, implicit - * alias ignored) — an alias directly abutting the star (no separator at all, comment or otherwise) - * must still be recognized as an implicit alias, not just one separated by whitespace or a - * comment. `SELECT *whatever, a FROM t` (a bare star with an abutting alias) is, by contrast, a - * genuine PostgreSQL syntax error — a bare `*` cannot itself take an alias — so this function's - * willingness to call it a star for an empty qualifier is unreachable on real input, not a gap - * that needs closing. + * On PostgreSQL 18.4, `SELECT u.*whatever, preferences FROM users u` returns 5 columns (star + * expands, implicit alias ignored): an alias directly abutting the star, with no separator at all, + * must still be recognized as an implicit alias. * - * Not claimed exhaustive: [parseSelectItems] has no independent real-column-count to cross-check - * against at the point it runs, so a spelling this function fails to recognize there degrades - * silently to a wrong, shifted mapping rather than a fail-safe. + * Not claimed exhaustive: [parseSelectItems] has no independent real-column-count to check this + * function's answer against, so an item shape this function fails to recognize degrades silently + * to a wrong, shifted mapping rather than a fail-safe. */ internal fun isStarItem(item: String): Boolean { val text = stripCommentsAndWhitespace(item.trim()) @@ -215,15 +149,10 @@ private fun unwrapWrappingParentheses(text: StrippedText): StrippedText { * Finds where a trailing implicit alias starts in [text] (already normalized by * [stripCommentsAndWhitespace]), or `null` if there is none. * - * Walks [text] forward, recording the start and end of the last segment seen (see - * [matchTrailingAliasSegment] for what counts as one). A character that doesn't start a segment, - * and a lexical token that isn't a segment (a single-quoted string literal, a dollar-quoted - * string — skipped via [skipLexicalToken]), are walked over without ending the search; they simply - * mean the segment recorded so far is not the final one. An implicit alias exists only if the last - * recorded segment ends exactly at `text.length` (nothing trails it) and starts at an index - * greater than `0` (there is something — the qualifier and its star — before it; a segment - * spanning the entire text is not "prefix plus alias", it's just one bare identifier with no star - * in it at all, e.g. `preferencesprefs`). + * An implicit alias exists only if the last segment found (see [matchTrailingAliasSegment] for + * what counts as one) ends exactly at `text.length` and starts at an index greater than `0` — a + * segment spanning the entire text is just one bare identifier with no star in it at all (e.g. + * `preferencesprefs`), not a qualifier plus an alias. * * @return The index where the trailing alias segment starts, or `null` if [text] has no such * segment. @@ -249,23 +178,14 @@ private fun findTrailingImplicitAliasStart(text: StrippedText): Int? { /** * The expression/alias split of an item with a trailing implicit (no-`AS`) alias — e.g. * `UPPER(a) y` splits into expression `UPPER(a)` and alias `y` — or `null` if [item] has no such - * trailing alias at all, generalizing [findTrailingImplicitAliasStart] (the same detection - * [isStarItem] uses for a star's own trailing alias) beyond star items to any item's text. - * - * That function already answers only for a trailing segment that is the item's final token and - * does not span the entire item, so a bare column reference (`description` — one segment covering - * the whole text) correctly returns `null` here, not itself, matching this function's own "no - * implicit alias" contract for that shape. + * trailing alias (including a bare column reference like `description`, one segment spanning the + * whole item). * - * [item] is stripped of comments/whitespace only to locate the split point ([findTrailingImplicitAliasStart] - * needs that normalized form) — the returned [ItemAndImplicitAlias.expression] is sliced out of - * [item] itself, original formatting (including any comment [stripComments] must still remove - * downstream) intact, via [StrippedText.originalIndexOf] translating the stripped split point back - * to [item]'s own indices. + * The returned [ItemAndImplicitAlias.expression] is sliced out of [item] itself, original + * formatting (including any comment) intact. * * @param item The full item text — expression and any trailing implicit alias together, with no - * `AS` keyword having already been found and split off (an item with an explicit `AS` alias is - * never passed here; its alias is [extractAlias]'s own, unrelated concern). + * `AS` keyword already found and split off. */ internal fun splitTrailingImplicitAlias(item: String): ItemAndImplicitAlias? { val stripped = stripCommentsAndWhitespace(item) @@ -284,21 +204,14 @@ internal data class ItemAndImplicitAlias(val expression: String, val alias: Stri /** * Matches one segment starting at [start] in [text], in this precedence order: - * 1. A Unicode-escape identifier — `U&`/`u&` immediately followed by a double-quoted identifier - * (via [skipLexicalToken]), optionally followed by the word `UESCAPE` and a single-quoted - * escape-character string, in which case the segment extends through that string — see - * [matchUnicodeEscapeIdentifierSegment]. Tried first: for `u.*U&"a"`, checking the bare-identifier - * rule (3, below) first would match `U` alone as a one-character identifier segment, then - * `"a"` as a separate later segment — the alias would appear to end at `"a"`, but the prefix - * would wrongly include the dangling `U&`, and the star would never be found as `.` + `*` - * immediately before it. Trying the Unicode-escape rule first consumes `U&"a"` as one segment, - * so the star at `.*` immediately precedes it, exactly as PostgreSQL itself parses it. - * 2. A bare double-quoted identifier, `"..."` (`""`-doubling included, via [skipLexicalToken]). - * 3. An unquoted identifier: first character [isIdentifierStartChar] (a letter, `_`, or any - * character whose code is `>= 0x80`), every subsequent character [isIdentifierChar] — - * PostgreSQL identifiers may not start with a digit or `$`. A `$` encountered mid-run stops the - * run instead of continuing it, per [OriginalAdjacency]'s own gate, when [text] says it was not - * genuinely adjacent to the character before it — see the loop below. + * 1. A Unicode-escape identifier — `U&`/`u&` immediately followed by a double-quoted identifier, + * optionally extended through a `UESCAPE ''` clause (see + * [matchUnicodeEscapeIdentifierSegment]). Tried first: for `u.*U&"a"`, matching the bare + * identifier rule (3, below) first would consume `U` alone as a segment, then `"a"` as a + * separate later segment, leaving a dangling `U&` attached to the prefix and hiding the star. + * 2. A bare double-quoted identifier, `"..."` (`""`-doubling included). + * 3. An unquoted identifier: first character [isIdentifierStartChar], every subsequent character + * [isIdentifierChar] — PostgreSQL identifiers may not start with a digit or `$`. * * @return The index immediately after the matched segment, or `null` if [start] does not begin * one. @@ -310,31 +223,20 @@ private fun matchTrailingAliasSegment(text: StrippedText, start: Int): Int? { val afterToken = text.skipLexicalToken(start) return if (afterToken != start) afterToken else null } - // PostgreSQL's lexer admits any byte >= 0x80 to start an unquoted identifier too (not merely to - // continue one, which [isIdentifierChar] already covers) — not merely a - // Unicode `isLetter()`. A combining mark (an alias written in NFD, e.g. "préfs" spelled - // p-r-e-COMBINING_ACUTE-f-s), a symbol (a currency sign `€`, `©`, `°`), and a supplementary-plane - // character (an astral emoji `🚀`, a mathematical alphanumeric symbol `𝐀`) are all legal first - // characters of an unquoted identifier that PostgreSQL accepts, none of which `isLetter()` - // recognizes as a letter — `isLetter()` alone therefore missed every one of those alias shapes, - // the dangerous direction (see [isStarItem]'s KDoc). A surrogate pair (as a supplementary-plane - // character always is, in a Kotlin/UTF-16 `String`) is naturally covered without special - // handling: both of its code units are >= 0x80, so the ordinary per-character loop below - // consumes each half in turn. isIdentifierStartChar is the shared predicate for this rule — see - // its KDoc for the other call sites. + // PostgreSQL's lexer admits any byte >= 0x80 to start an unquoted identifier, not merely a + // Unicode `isLetter()`: a combining mark (an alias written in NFD, e.g. "préfs" spelled + // p-r-e-COMBINING_ACUTE-f-s), a currency sign (`€`), and a supplementary-plane character (an + // astral emoji, a mathematical alphanumeric symbol like `𝐀`) are all legal first characters that + // `isLetter()` does not recognize as letters. A surrogate pair is covered without special + // handling, since both of its code units are >= 0x80. if (!isIdentifierStartChar(text[start])) return null var i = start + 1 while (i < text.length && isIdentifierChar(text[i]) && text.wereAdjacent(i - 1)) { // Gated on adjacency for every continuation character, not just "$": stripping deletes the - // whitespace/comment that used to separate two independent identifiers PostgreSQL itself lexed - // apart -- "description dx" strips to "descriptiondx", and without this gate the run would - // swallow both tokens into one fused segment spanning the whole text, tripping - // findTrailingImplicitAliasStart's "a segment spanning the entire text is not an alias" guard so - // no alias is found at all. Stopping at the first non-adjacent character ends the first - // identifier's segment at its real token boundary, letting findTrailingImplicitAliasStart pick - // the second identifier up as its own later segment. This also covers "$": a non-adjacent "$" - // stops the run here and is handed back to [StrippedText.skipLexicalToken], which recognizes a - // dollar-quoted string as one opaque token and walks over it whole. + // separator between two independently-lexed identifiers, so without this gate the run would + // fuse both into one segment spanning the whole text and no alias would be found at all. A + // non-adjacent "$" is handed back to [StrippedText.skipLexicalToken], which recognizes a + // dollar-quoted string as one opaque token. i++ } return i @@ -343,22 +245,15 @@ private fun matchTrailingAliasSegment(text: StrippedText, start: Int): Int? { /** * Matches a Unicode-escape identifier — `U&`/`u&` immediately followed by a double-quoted * identifier, e.g. `U&"my*table"` — starting at [start] in [text], optionally extended by a - * `UESCAPE ''` clause naming a custom escape character (PostgreSQL merges the identifier, - * the `UESCAPE` keyword, and the single-quoted escape-character string into one lexical unit — - * `U&"d!0061t" UESCAPE '!'` and `U&"!0074" UESCAPE '!'` are each a single identifier, - * both resolving via the `!`-escape to the same characters `U&"data"`/`U&"t"` would spell without - * one). [text] has already had all whitespace removed by [stripCommentsAndWhitespace], so the - * `UESCAPE` keyword and its string abut the identifier directly with no separator to skip. + * `UESCAPE ''` clause naming a custom escape character. PostgreSQL merges the identifier, + * the `UESCAPE` keyword, and the single-quoted escape-character string into one lexical unit: + * `U&"d!0061t" UESCAPE '!'` and `U&"!0074" UESCAPE '!'` are each a single identifier, both + * resolving via the `!`-escape to the same characters `U&"data"`/`U&"t"` would spell without one. * - * The `UESCAPE` keyword's own adjacency to the identifier before it is deliberately left ungated, - * unlike every other multi-character adjacency decision in this file: PostgreSQL itself permits - * whitespace between a `U&"..."` identifier and its `UESCAPE` clause (`U&"!0074" - * UESCAPE '!'` and `U&"!0074"UESCAPE'!'` both resolve identically), so the abutment stripping - * creates here is not a manufactured token — [stripCommentsAndWhitespace] is implementing the real - * grammar, not accidentally fusing two things PostgreSQL lexed apart. And even if some other - * adjacency this function doesn't check turned out to matter, a false match here only extends the - * matched segment further than it should — the same safe direction [isStarItem] relies on - * throughout (see its KDoc), not the dangerous one a gate exists to prevent. + * The `UESCAPE` keyword's adjacency to the identifier before it is left ungated: PostgreSQL itself + * permits whitespace there (`U&"!0074" UESCAPE '!'` and `U&"!0074"UESCAPE'!'` both resolve + * identically), so stripping that whitespace does not manufacture a token PostgreSQL didn't + * already treat as one unit. * * @return The index immediately after the identifier (and its `UESCAPE` clause, if present), or * `null` if [start] does not begin a `U&`/`u&`-prefixed double-quoted identifier at all. @@ -385,36 +280,24 @@ private fun matchUnicodeEscapeIdentifierSegment(text: StrippedText, start: Int): /** * Checks that [qualifierEndingInDot] (the qualifier before a star recognized on path 2 of - * [isStarItem], guaranteed by its caller to end in `.`) is acceptable. Deliberately not an - * enumeration of every character that may legitimately precede the dot (`"`, `)`, `]`, an - * identifier run, a `DISTINCT ON (...)` prefix, a parenthesized composite expansion, an array - * subscript, a Unicode-escape identifier with a `UESCAPE` clause...): `false` is the dangerous - * answer here (see [isStarItem]'s KDoc), so it must be earned by an actual disqualifying shape, - * not handed out by default whenever a new qualifier shape isn't yet on an enumerated list. + * [isStarItem], guaranteed by its caller to end in `.`) is acceptable. * * Rejects only when the run of [isIdentifierChar] characters immediately preceding the final `.` * is non-empty and its first character is an ASCII digit (`'0'..'9'`, not `Char.isDigit()`) — a - * digit-leading run before a dot is a numeric literal (`2.` in `SELECT 2.*3 lbl, a FROM t` — - * arithmetic returning 2 columns, not a star), the one lexical ambiguity a real - * qualifying dot has, and PostgreSQL numeric literals use ASCII digits exclusively — so a - * non-ASCII digit (Unicode category Nd, e.g. `٣` ARABIC-INDIC DIGIT THREE, `３` FULLWIDTH DIGIT - * THREE) can only be an identifier's first character there, never a numeral: a real - * table literally named `٣`, `SELECT ٣.* x, a FROM ٣` returns 3 columns. `Char.isDigit()` is - * Unicode-aware and would wrongly reject that qualifier as if it were numeric. + * digit-leading run before a dot is a numeric literal (`2.` in `SELECT 2.*3 lbl, a FROM t`, which + * returns 2 columns: arithmetic, not a star). `Char.isDigit()` is Unicode-aware and would wrongly + * reject a qualifier starting with a non-ASCII digit (Unicode category Nd) as if it were numeric — + * with a table literally named `٣` (ARABIC-INDIC DIGIT THREE), `SELECT ٣.* x, a FROM ٣` returns + * 3 columns. * - * The run scan uses [isIdentifierChar] — the same predicate every identifier-continuation check in - * this file shares (see its KDoc), including [matchTrailingAliasSegment]'s own character class — - * rather than a narrower letter-or-digit-only check: a `>= 0x80` character that is not a letter or - * digit (`€`, `©`, `¹`) would otherwise truncate the run early, leaving whatever ASCII digit sits - * before it looking like the run's own start (`x€9.`: scanning backward with a letter-or-digit-only - * check stops at `€`, making `9` look like the run's start and wrongly rejecting the whole thing as - * numeric, when the real run is `x€9` — letter-led, and correctly acceptable). Sharing one - * predicate makes that symmetry structural rather than a comment to remember to keep in sync. + * The run scan uses [isIdentifierChar] rather than a narrower letter-or-digit-only check: a + * `>= 0x80` character that is not a letter or digit (`€`) would otherwise truncate the run early, + * making an ASCII digit before it look like the run's own start: scanning `x€9.` backward with a + * letter-or-digit-only check stops at `€`, so `9` looks like the run's start and the qualifier is + * wrongly rejected as numeric, when the real run is `x€9` — letter-led, and acceptable. * * Accepts in every other case, including an empty run (the character immediately before the dot is - * `"`, `)`, `]`, or anything else that isn't an identifier-continuation character at all) — this - * is what lets a Unicode-escape qualifier like `U&"!0074"UESCAPE'!'.` (ending in the escape - * string's closing `'`) work with no special case for it whatsoever. + * `"`, `)`, `]`, or a Unicode-escape's closing `'`, as in `U&"!0074" UESCAPE '!'.`). */ private fun isStarQualifierAcceptable(qualifierEndingInDot: String): Boolean { val beforeDotIndex = qualifierEndingInDot.length - 2 diff --git a/generator/src/main/kotlin/norm/generator/SqlStatement.kt b/generator/src/main/kotlin/norm/generator/SqlStatement.kt index 5d91767d..ffa58742 100644 --- a/generator/src/main/kotlin/norm/generator/SqlStatement.kt +++ b/generator/src/main/kotlin/norm/generator/SqlStatement.kt @@ -15,7 +15,7 @@ import java.sql.ResultSet internal class SqlStatement( private val catalog: Catalog, private val query: Query, - private val generator: TypeRepository, + private val typeRepository: TypeRepository, ) { /** @@ -222,12 +222,12 @@ internal class SqlStatement( /** * Resolves the mappable type for a column with domain type support. */ - fun resolveMappableType(column: Column): SqlMappable = generator.resolveMappableType(column) + fun resolveMappableType(column: Column): SqlMappable = typeRepository.resolveMappableType(column) /** * Resolves the Kotlin [TypeName] for a column with domain type support. */ - fun resolveColumnType(column: Column): TypeName = generator.resolveColumnType(column) + fun resolveColumnType(column: Column): TypeName = typeRepository.resolveColumnType(column) private fun computeReturnType(): ReturnType { val queryResults = query.columns @@ -237,17 +237,17 @@ internal class SqlStatement( } else if (queryResults.size == 1 && queryResults.first().embedTable == null) { // The query returns a single column, so no wrapper is needed val column = queryResults.first() - val columnType = generator.resolveColumnType(column) + val columnType = typeRepository.resolveColumnType(column) ReturnType( columnType, - listOf(generator.resolveMappableType(column).resultSetAction(1)), + listOf(typeRepository.resolveMappableType(column).resultSetAction(1)), listOf(ParameterSpec(column.name, columnType)), ) } else if (isSingleTableStarProjection) { // The query is a star projection (eg SELECT * ...). Return a model of the table. - generator.getTypeProjectionForTable(starProjectionTable!!) + typeRepository.getTypeProjectionForTable(starProjectionTable!!) } else { - generator.buildTypeProjectionForQuery(query.name, queryResults, query.text) + typeRepository.buildTypeProjectionForQuery(query.name, queryResults, query.text) } } diff --git a/generator/src/main/kotlin/norm/generator/TypeRepository.kt b/generator/src/main/kotlin/norm/generator/TypeRepository.kt index ce4e262a..2e060e3f 100644 --- a/generator/src/main/kotlin/norm/generator/TypeRepository.kt +++ b/generator/src/main/kotlin/norm/generator/TypeRepository.kt @@ -1,6 +1,5 @@ package norm.generator -import com.squareup.kotlinpoet.ANY import com.squareup.kotlinpoet.ARRAY import com.squareup.kotlinpoet.ClassName import com.squareup.kotlinpoet.CodeBlock @@ -11,27 +10,17 @@ import com.squareup.kotlinpoet.ParameterizedTypeName.Companion.parameterizedBy import com.squareup.kotlinpoet.PropertySpec import com.squareup.kotlinpoet.TypeName import com.squareup.kotlinpoet.TypeSpec -import com.squareup.kotlinpoet.asClassName -import com.squareup.kotlinpoet.asTypeName -import java.math.BigDecimal -import java.sql.Blob -import java.time.Instant -import java.time.LocalDate -import java.time.LocalDateTime -import java.time.LocalTime -import java.time.OffsetDateTime -import java.time.OffsetTime -import java.util.UUID /** - * [JdbcTypeInfo] for Postgres enum types. + * [WireCodec] for Postgres enum types. * * Enum types require `setObject(index, value, Types.OTHER)` rather than `setString(index, value)`. * The Postgres JDBC driver rejects `VARCHAR` bindings for enum columns in prepared statements; - * `Types.OTHER` bypasses driver-side type enforcement and lets Postgres coerce the string. + * `Types.OTHER` bypasses driver-side type enforcement and lets Postgres coerce the string. This is + * the same codec instance the `json`/`jsonb` rows of [POSTGRES_BASE_TYPES] use, so the binding for + * a plain `json`/`jsonb` column and an enum column can never drift apart. */ -private val ENUM_JDBC_TYPE_INFO = - JdbcTypeInfo("getString", "setObject", false, "OTHER", useSqlTypeHint = true, kotlinType = String::class.asTypeName()) +private val ENUM_CODEC: WireCodec = POSTGRES_BASE_TYPES.getValue("json").codec /** * Repository for types generated as part of query generation. @@ -361,8 +350,8 @@ internal class TypeRepository( * Postgres domains (e.g., `CREATE DOMAIN email AS text`) are resolved to their base types * by analyzing query parameters. This method handles both standard types and domains. * - * Uses [SqlMappable.typeName] rather than [SqlMappable.klass] so that generated types - * (like enum classes) can provide their [TypeName] without requiring a [KClass] at generator time. + * Uses [SqlMappable.typeName], which generated types (like enum classes) can provide directly + * without needing a [kotlin.reflect.KClass] at generator time. * * Array wrapping is handled by [tryResolveStandardType] which returns an [ArrayTypeDecorator] * whose [SqlMappable.typeName] is already the correct parameterized array type. @@ -422,7 +411,7 @@ internal class TypeRepository( ): SqlMappable { val applicationTypeName = parseTypeName(mapping.kotlinType) val adapterPropertyName = userAdapterPropertyName(mapping) - val jdbcTypeInfo = resolveJdbcTypeInfoForType(postgresType) + val codec = resolveWireCodecForType(postgresType) ?: error( "Postgres type '$postgresType' cannot be used with a custom adapter — " + "no JDBC type mapping is available.", @@ -436,21 +425,21 @@ internal class TypeRepository( postgresTypeName = postgresType, ) } - return AdaptedTypeSqlMappable(applicationTypeName, adapterPropertyName, notNull, jdbcTypeInfo) + return AdaptedTypeSqlMappable(applicationTypeName, adapterPropertyName, notNull, codec) } /** - * Resolves [JdbcTypeInfo] for any Postgres type, chaining through enums and domains as needed. + * Resolves a [WireCodec] for any Postgres type, chaining through enums and domains as needed. * * - Enum types → String (VARCHAR) * - Domain types → chains to the domain's base type - * - Standard types → uses [resolveJdbcTypeInfo] + * - Standard types → uses [resolveWireCodec] */ - private fun resolveJdbcTypeInfoForType(postgresType: String): JdbcTypeInfo? { - if (postgresType in enumsByName) return ENUM_JDBC_TYPE_INFO + private fun resolveWireCodecForType(postgresType: String): WireCodec? { + if (postgresType in enumsByName) return ENUM_CODEC val domain = domainsByName[postgresType] - if (domain != null) return resolveJdbcTypeInfoForType(domain.baseType) - return resolveJdbcTypeInfo(postgresType) + if (domain != null) return resolveWireCodecForType(domain.baseType) + return resolveWireCodec(postgresType) } /** @@ -474,7 +463,7 @@ internal class TypeRepository( postgresTypeName = typeName, ) } - return AdaptedTypeSqlMappable(enumClassName, propertyName, notNull, ENUM_JDBC_TYPE_INFO) + return AdaptedTypeSqlMappable(enumClassName, propertyName, notNull, ENUM_CODEC) } /** Returns the [SqlMappable] for a standard Postgres type, or `null` if not recognized. */ @@ -492,7 +481,7 @@ internal class TypeRepository( // 4294967295 is rejected by Postgres with "value out of range". Callers must keep bound values // within `0..4294967295` themselves; this mapping does not validate that range. if (typeName == "oid" || typeName == "pg_catalog.oid") { - val elementType = JdbcTypes.LONG.decorateForNullable(notNull = false) + val elementType = ScalarSqlMappable(POSTGRES_BASE_TYPES.getValue("int8").codec, notNull = false) val arrayTypeName = ARRAY.parameterizedBy(elementType.typeName.copy(nullable = true)) .copy(nullable = !notNull) return ArrayTypeDecorator(elementType, arrayTypeName, postgresArrayElementTypeName(typeName)) @@ -500,7 +489,7 @@ internal class TypeRepository( // Postgres array elements are always nullable regardless of the column's NOT NULL constraint, // so the element read must be the nullable form: getInt would turn a NULL element into 0, and - // InstantSqlMappable's non-null read would throw NullPointerException on one. + // InstantViaOffsetDateTimeCodec's non-null read would throw NullPointerException on one. val elementType = resolveBaseType(typeName, notNull = false) ?: return null val arrayTypeName = ARRAY.parameterizedBy(elementType.typeName.copy(nullable = true)) @@ -514,10 +503,10 @@ internal class TypeRepository( * For scalar columns, returns [AdaptedTypeSqlMappable]. For array columns (e.g., `email[]`), * returns [AdaptedArrayTypeSqlMappable] which generates per-element adapter decode/encode calls. * - * [resolveJdbcTypeInfo] covers every key in [BASE_TYPE_RESOLVERS] (enforced by - * [ColumnTypeMappingTest]'s domain-base-type-parity sweep), so `error` below is unreachable for a - * domain over a common base type like `timestamptz` or `uuid` — see [domainKotlinBaseType]'s - * KDoc for the (intentional) case where it remains reachable. + * [resolveWireCodec] and [resolveBaseType] both read [POSTGRES_BASE_TYPES], so `error` below + * is unreachable, by construction, for a domain over any base type that map supports (e.g. + * `timestamptz` or `uuid`) — see [domainKotlinBaseType]'s KDoc for the (intentional) case where + * it remains reachable. */ private fun tryResolveDomainType(typeName: String, notNull: Boolean, isArray: Boolean): SqlMappable? { val domain = domainsByName[typeName] ?: return null @@ -525,7 +514,7 @@ internal class TypeRepository( val domainClassName = ClassName(packageName, domain.name.snakeToCamelCase().titleCase()) val propertyName = domainAdapterPropertyName(domain) - val jdbcTypeInfo = resolveJdbcTypeInfo(domain.baseType) + val codec = resolveWireCodec(domain.baseType) ?: error("Domain ${domain.name} has unsupported base type: ${domain.baseType}") if (isArray) { @@ -536,610 +525,17 @@ internal class TypeRepository( postgresTypeName = typeName, ) } - return AdaptedTypeSqlMappable(domainClassName, propertyName, notNull, jdbcTypeInfo) + return AdaptedTypeSqlMappable(domainClassName, propertyName, notNull, codec) } /** * Maps a Postgres type name to its base [SqlMappable], or `null` if not recognized. * * [typeName] may carry a `pg_catalog.` qualification (e.g. `pg_catalog.int4`); it is stripped - * once here rather than duplicated per literal in [BASE_TYPE_RESOLVERS], so every entry in that + * once here rather than duplicated per literal in [POSTGRES_BASE_TYPES], so every entry in that * map accepts both the qualified and unqualified spelling without needing its own branch for * each. */ private fun resolveBaseType(typeName: String, notNull: Boolean): SqlMappable? = - BASE_TYPE_RESOLVERS[typeName.removePrefix("pg_catalog.")]?.invoke(notNull) -} - -/** - * Every canonical Postgres base type name [TypeRepository.resolveBaseType] accepts, keyed by name - * (after stripping a `pg_catalog.` qualification — see [TypeRepository.resolveBaseType]) and - * mapped to the [SqlMappable] used for a plain (non-domain) column of that type. - * - * This is the single source of truth for "every type name resolveBaseType accepts": both - * [TypeRepository.resolveBaseType] itself and [ColumnTypeMappingTest]'s domain-base-type-parity - * sweep read from these exact keys, so a type added here without a matching [resolveJdbcTypeInfo] - * entry fails that sweep immediately — see [resolveJdbcTypeInfo]'s KDoc for the invariant this - * enforces between the two maps. - * - * Includes the `serial`/`smallserial`/`bigserial` pseudo-types even though Postgres rejects - * `CREATE DOMAIN ... AS serial` outright (`type "serial" does not exist` on a live - * server; a domain's base is always a real, registered `pg_type`, so `domain.baseType` can never - * actually be one of these), and even though `resolveJdbcTypeInfo`'s only other callers - * ([TypeRepository.buildUserConfiguredMappable]'s user-configured type mappings) also only ever - * see the real, JDBC-reported type name, never a serial alias: keeping them out would make this - * map a proper subset of `resolveBaseType`'s branches, silently reintroducing exactly the kind of - * incomplete "should be identical" list this map exists to prevent. - */ -internal val BASE_TYPE_RESOLVERS: Map SqlMappable> = mapOf( - "smallserial" to { notNull: Boolean -> JdbcTypes.SHORT.decorateForNullable(notNull) }, - "serial2" to { notNull: Boolean -> JdbcTypes.SHORT.decorateForNullable(notNull) }, - "serial" to { notNull: Boolean -> JdbcTypes.INT.decorateForNullable(notNull) }, - "serial4" to { notNull: Boolean -> JdbcTypes.INT.decorateForNullable(notNull) }, - "bigserial" to { notNull: Boolean -> JdbcTypes.LONG.decorateForNullable(notNull) }, - "serial8" to { notNull: Boolean -> JdbcTypes.LONG.decorateForNullable(notNull) }, - - "smallint" to { notNull: Boolean -> JdbcTypes.SHORT.decorateForNullable(notNull) }, - "int2" to { notNull: Boolean -> JdbcTypes.SHORT.decorateForNullable(notNull) }, - "integer" to { notNull: Boolean -> JdbcTypes.INT.decorateForNullable(notNull) }, - "int" to { notNull: Boolean -> JdbcTypes.INT.decorateForNullable(notNull) }, - "int4" to { notNull: Boolean -> JdbcTypes.INT.decorateForNullable(notNull) }, - "bigint" to { notNull: Boolean -> JdbcTypes.LONG.decorateForNullable(notNull) }, - "int8" to { notNull: Boolean -> JdbcTypes.LONG.decorateForNullable(notNull) }, - - "real" to { notNull: Boolean -> JdbcTypes.FLOAT.decorateForNullable(notNull) }, - "float4" to { notNull: Boolean -> JdbcTypes.FLOAT.decorateForNullable(notNull) }, - "float" to { notNull: Boolean -> JdbcTypes.DOUBLE.decorateForNullable(notNull) }, - "double precision" to { notNull: Boolean -> JdbcTypes.DOUBLE.decorateForNullable(notNull) }, - "float8" to { notNull: Boolean -> JdbcTypes.DOUBLE.decorateForNullable(notNull) }, - "numeric" to { _: Boolean -> JdbcTypes.BIG_DECIMAL }, - - "bool" to { notNull: Boolean -> JdbcTypes.BOOLEAN.decorateForNullable(notNull) }, - "boolean" to { notNull: Boolean -> JdbcTypes.BOOLEAN.decorateForNullable(notNull) }, - - // Not JdbcTypes.STRING: pgjdbc rejects setString() for json and jsonb columns. - "json" to { notNull: Boolean -> JsonSqlMappable(notNull) }, - "jsonb" to { notNull: Boolean -> JsonSqlMappable(notNull) }, - - // Scalar oid maps to Blob: pgjdbc's setBlob() creates a Postgres large object and stores its - // oid, the standard large-object convention. oid[] does not share this mapping (see - // tryResolveStandardType) because an array of large-object handles has no coherent JDBC - // semantics, and real-world oid[] columns hold plain catalog identifiers, not large objects. - "oid" to { _: Boolean -> JdbcTypes.BLOB }, - "bytea" to { _: Boolean -> PostgresSupportedTypes.BYTE_ARRAY }, - - "date" to { _: Boolean -> PostgresSupportedTypes.LOCAL_DATE }, - "time" to { _: Boolean -> PostgresSupportedTypes.LOCAL_TIME }, - "timetz" to { _: Boolean -> PostgresSupportedTypes.OFFSET_TIME }, - "timestamp" to { _: Boolean -> PostgresSupportedTypes.LOCAL_DATE_TIME }, - "timestamptz" to { notNull: Boolean -> InstantSqlMappable(notNull) }, - - "text" to { _: Boolean -> JdbcTypes.STRING }, - "varchar" to { _: Boolean -> JdbcTypes.STRING }, - "bpchar" to { _: Boolean -> JdbcTypes.STRING }, - "string" to { _: Boolean -> JdbcTypes.STRING }, - - "uuid" to { _: Boolean -> PostgresSupportedTypes.UUID }, -) - -/** - * Canonicalizes a Postgres type name for use as the element type of - * [java.sql.Connection.createArrayOf]. - * - * The driver appends `[]` to this name and looks the result up in `pg_type`, so it must be a - * canonical `pg_type` name. [TypeRepository.resolveBaseType] additionally accepts SQL spellings - * (`integer`, `boolean`, `double precision`) and `pg_catalog.`-qualified names; without folding - * those here, `postgresArrayElementTypeName("integer")` would return `"integer"` verbatim and - * `createArrayOf` would fail with `Unable to find server array type for provided name {0}`, since - * `pg_type` has no row named `integer` — only `int4`. - * - * Every branch below was checked against a live PostgreSQL 17 server via - * `SELECT typname FROM pg_type WHERE oid = to_regtype(?)`: every alias here resolves to the - * canonical name on its right-hand side, and every `pg_catalog.`-qualified spelling of an - * already-canonical name (e.g. `pg_catalog.uuid`, `pg_catalog.timestamptz`) resolves to itself — - * confirming the universal `removePrefix` below is sufficient for those without a dedicated - * branch. `pg_catalog.boolean` and `pg_catalog.integer` do not resolve on a live server (`boolean` - * and `integer` are SQL-standard keyword aliases recognized only unqualified, not as schema- - * qualified `pg_catalog` names) — but that combination can never actually reach this function: - * JDBC's `TYPE_NAME`/`getColumnTypeName` always report the canonical, unqualified name. - * - * `serial` and its variants need no entry: Postgres has no serial array type, so a serial column - * can never reach the array path. - */ -internal fun postgresArrayElementTypeName(typeName: String): String = - when (val canonical = typeName.removePrefix("pg_catalog.")) { - "smallint" -> "int2" - "integer", "int" -> "int4" - "bigint" -> "int8" - "real" -> "float4" - "double precision", "float" -> "float8" - "boolean" -> "bool" - "string" -> "text" - else -> canonical - } - -/** - * Maps a Postgres base type name to its [JdbcTypeInfo], or returns `null` if unsupported. - * - * Every key in [BASE_TYPE_RESOLVERS] has an entry here — [ColumnTypeMappingTest]'s domain-base- - * type-parity sweep asserts this directly, rather than relying on the two lists being hand-kept in - * sync, so a domain over any base type [TypeRepository.resolveBaseType] itself supports (e.g. - * `CREATE DOMAIN d AS timestamptz`) always resolves here too: [TypeRepository]'s domain resolution - * chains through this function (see [TypeRepository.tryResolveDomainType] and - * [domainKotlinBaseType][norm.generator.domainKotlinBaseType]), and its `error()` calls are - * reachable only for a base type [TypeRepository.resolveBaseType] itself does not support either - * (e.g. `xml`, `interval`, `money` — Postgres allows a domain over any of these, but Norm has never - * mapped them to a Kotlin type as a plain column type, so the same limitation applies to a domain - * built on one). That failure is intentional: a clear, immediate `error()` naming the unsupported - * type is preferable to silently guessing a mapping for a type Norm has no tested behavior for. - * - * Every getter/setter/Kotlin-type combination below matches [BASE_TYPE_RESOLVERS]'s NON-domain - * mapping for the same key exactly — see [JdbcTypeInfo.getterClassHint] and - * [JdbcTypeInfo.convertOffsetDateTimeToInstant] for the cases (`java.time` types, `uuid`, and - * `timestamptz` specifically) where matching the non-domain path requires more than a plain - * `getX`/`setX` method pair, each checked against pgjdbc 42.7.13's source rather than assumed. - */ -internal fun resolveJdbcTypeInfo(baseTypeName: String): JdbcTypeInfo? = when (baseTypeName) { - "smallserial", "serial2", "smallint", "int2" -> - JdbcTypeInfo("getShort", "setShort", true, "SMALLINT", kotlinType = Short::class.asTypeName()) - "serial", "serial4", "integer", "int", "int4" -> - JdbcTypeInfo("getInt", "setInt", true, "INTEGER", kotlinType = Int::class.asTypeName()) - "bigserial", "serial8", "bigint", "int8" -> - JdbcTypeInfo("getLong", "setLong", true, "BIGINT", kotlinType = Long::class.asTypeName()) - "real", "float4" -> - JdbcTypeInfo("getFloat", "setFloat", true, "REAL", kotlinType = Float::class.asTypeName()) - "float", "double precision", "float8" -> - JdbcTypeInfo("getDouble", "setDouble", true, "DOUBLE", kotlinType = Double::class.asTypeName()) - "bool", "boolean" -> - JdbcTypeInfo("getBoolean", "setBoolean", true, "BOOLEAN", kotlinType = Boolean::class.asTypeName()) - "numeric" -> - JdbcTypeInfo("getBigDecimal", "setBigDecimal", false, "NUMERIC", kotlinType = BigDecimal::class.asTypeName()) - // json and jsonb require setObject(..., Types.OTHER): Postgres JDBC rejects setString() for both - // in prepared statements, just as it does for enum columns. Keep in sync with JsonSqlMappable, - // which defines the same binding for plain (adapterless) json and jsonb columns. - "json", "jsonb" -> - JdbcTypeInfo( - "getString", - "setObject", - false, - "OTHER", - useSqlTypeHint = true, - kotlinType = String::class.asTypeName(), - ) - "text", "varchar", "bpchar", "string" -> - JdbcTypeInfo("getString", "setString", false, "VARCHAR", kotlinType = String::class.asTypeName()) - // Matches JdbcTypes.BLOB, the non-domain scalar mapping for oid (see BASE_TYPE_RESOLVERS): - // pgjdbc's getBlob()/setBlob() are plain named methods, needing no class-hint or Types constant. - "oid" -> - JdbcTypeInfo("getBlob", "setBlob", false, "BLOB", kotlinType = Blob::class.asTypeName()) - // Matches PostgresSupportedTypes.BYTE_ARRAY: java.sql.ResultSet.getBytes/PreparedStatement.setBytes - // are plain named methods for bytea, needing no class-hint. - "bytea" -> - JdbcTypeInfo("getBytes", "setBytes", false, "BINARY", kotlinType = ByteArray::class.asTypeName()) - // Matches PostgresSupportedTypes.LOCAL_DATE/LOCAL_TIME/OFFSET_TIME/LOCAL_DATE_TIME: pgjdbc's - // plain getObject(int) returns java.sql.Date/Time/Timestamp for these columns, not the java.time - // type, so the read needs the class-qualified getObject(int, Class) overload (getterClassHint). - // The write side needs no such qualification: PgPreparedStatement.setObject(int, Object) already - // dispatches on the runtime type of a LocalDate/LocalTime/OffsetTime/LocalDateTime/OffsetDateTime - // argument directly (pgjdbc 42.7.13's source). - "date" -> - JdbcTypeInfo( - "getObject", - "setObject", - false, - "DATE", - kotlinType = LocalDate::class.asTypeName(), - getterClassHint = LocalDate::class.asClassName(), - ) - "time" -> - JdbcTypeInfo( - "getObject", - "setObject", - false, - "TIME", - kotlinType = LocalTime::class.asTypeName(), - getterClassHint = LocalTime::class.asClassName(), - ) - "timetz" -> - JdbcTypeInfo( - "getObject", - "setObject", - false, - "TIME_WITH_TIMEZONE", - kotlinType = OffsetTime::class.asTypeName(), - getterClassHint = OffsetTime::class.asClassName(), - ) - "timestamp" -> - JdbcTypeInfo( - "getObject", - "setObject", - false, - "TIMESTAMP", - kotlinType = LocalDateTime::class.asTypeName(), - getterClassHint = LocalDateTime::class.asClassName(), - ) - // Matches InstantSqlMappable: the wire representation is OffsetDateTime (read via the - // class-qualified getObject, written via plain setObject — both checked against pgjdbc's - // source the same way as the other java.time entries above), but the Kotlin representation the - // non-domain scalar path uses is Instant, via a `.toInstant()`/`OffsetDateTime.ofInstant(...)` - // conversion — see JdbcTypeInfo.convertOffsetDateTimeToInstant's KDoc. - "timestamptz" -> - JdbcTypeInfo( - "getObject", - "setObject", - false, - "TIMESTAMP_WITH_TIMEZONE", - kotlinType = Instant::class.asTypeName(), - getterClassHint = OffsetDateTime::class.asClassName(), - convertOffsetDateTimeToInstant = true, - ) - // Matches PostgresSupportedTypes.UUID: java.sql.ResultSet.getObject(int) is declared to return - // Object, so a bare getObject(index) call is statically Any in Kotlin regardless of what pgjdbc - // returns at runtime — PgResultSet's internalGetObject does special-case the Postgres "uuid" - // type by name and hands back a java.util.UUID instance (per pgjdbc 42.7.13's - // source), but that's a runtime fact, not a static type, and ColumnAdapter.decode requires a statically-typed UUID argument. The class-qualified - // getObject(int, Class) overload (getterClassHint) fixes the static type; pgjdbc's - // PgResultSet#getObject(int, Class) explicitly special-cases `type == UUID.class` by - // delegating to the same runtime read and casting, so this is safe. - "uuid" -> - JdbcTypeInfo( - "getObject", - "setObject", - false, - "OTHER", - kotlinType = UUID::class.asTypeName(), - getterClassHint = UUID::class.asClassName(), - ) - else -> null -} - -/** - * Describes a property's source in the database. - * - * @property propertyName The Kotlin property name. - * @property comment The Postgres column comment. Empty if none. - * @property sourceTable The database table the column comes from. `null` for computed columns. - * @property sourceColumn The original column name in the database. `null` for computed columns. - * @property expression The SQL expression for computed columns (e.g. `COUNT(*)`). Empty if not computed. - */ -internal data class PropertySource( - val propertyName: String, - val comment: String, - val sourceTable: String?, - val sourceColumn: String?, - val expression: String = "", -) - -/** - * Adds a class-level KDoc block with an optional description, table mapping, and `@property` tags. - * - * Produces a single consolidated KDoc block rather than separate per-property doc comments, which is the - * idiomatic Kotlin style for data classes with constructor properties. - * - * For table projections, the table name is shown as "Maps to the `X` table.". - * For query projections, the SQL is included and source columns are shown per-property as `table.column` references. - * - * @param classComment The table or class-level comment. May be empty. - * @param tableName The database table this class fully maps to. `null` for ad-hoc query projections. - * @param properties Source information for each property. - * @param sql The SQL query text. Included in query projection KDoc as a fenced code block. - * @param reservedWords The connected PostgreSQL server's reserved keywords, forwarded to - * [quoteSqlIdentifierIfNeeded] for each property's `` `table.column` `` source reference. Empty - * for a table projection, which never renders a source reference at all. - */ -internal fun TypeSpec.Builder.addClassKdoc( - classComment: String, - tableName: String?, - properties: List, - sql: String = "", - reservedWords: Set = emptySet(), -) { - val hasTableMapping = tableName != null - // KotlinPoet's KDoc emission rewrites "/*"/"*/" to "/*"/"*/" inside every KDoc block so a - // literal block comment can't prematurely close the surrounding "/** ... */" comment, but CommonMark - // never decodes that HTML entity back inside a fenced code block -- see - // containsUnescapableBlockCommentDelimiter. Declining the whole fenced block is the only correct - // choice for a query containing either sequence. - val canRenderSqlVerbatim = sql.isNotEmpty() && !containsUnescapableBlockCommentDelimiter(sql) - // A property whose name can't be rendered as a `@property` name token at all - // (formatAsKdocPropertyReference returns `null`) is dropped here rather than emitted with a - // mangled name that reads back as a different property than the one actually declared. - val documentedProperties = properties.mapNotNull { property -> - if (!property.hasDocumentation(hasTableMapping, reservedWords)) return@mapNotNull null - val formattedName = property.propertyName.formatAsKdocPropertyReference() ?: return@mapNotNull null - formattedName to property - } - if (classComment.isEmpty() && !hasTableMapping && !canRenderSqlVerbatim && documentedProperties.isEmpty()) return - - val kdoc = buildString { - if (classComment.isNotEmpty()) { - append(classComment) - } - if (hasTableMapping) { - if (isNotEmpty()) append("\n\n") - append("Maps to the `$tableName` table.") - } - if (canRenderSqlVerbatim) { - if (isNotEmpty()) append("\n\n") - // A fixed 3-backtick fence breaks if sql itself contains a run of 3+ backticks -- a line - // matching or exceeding the fence's own length terminates the fenced block early. A fence one - // backtick longer than any run already in sql can never be mistaken for a closing fence. - val fence = markdownFenceDelimiter(sql) - append(fence).append("sql\n") - append(sql.trim()) - append("\n").append(fence) - } - if (documentedProperties.isNotEmpty()) { - if (isNotEmpty()) append("\n\n") - for ((index, formattedNameAndProperty) in documentedProperties.withIndex()) { - val (formattedName, property) = formattedNameAndProperty - append("@property $formattedName ") - if (property.comment.isNotEmpty()) { - // Every `@property` line shares one CommonMark paragraph (no blank line between them), so - // an unescaped backtick in one comment could pair with a later property's own - // source-reference span instead of closing here. Escaping it keeps it from ever being read - // as a code-span delimiter. - append(escapeMarkdownBacktick(property.comment)) - } - if (!hasTableMapping) { - val source = property.sourceReference(reservedWords) - if (source != null) { - if (property.comment.isNotEmpty()) append(" ") - append("($source)") - } - } - if (index < documentedProperties.lastIndex) append("\n") - } - } - } - addKdoc("%L", kdoc) -} - -/** - * Whether this property has any documentation to show in KDoc. - */ -private fun PropertySource.hasDocumentation(hasTableMapping: Boolean, reservedWords: Set): Boolean = - comment.isNotEmpty() || (!hasTableMapping && sourceReference(reservedWords) != null) - -/** - * Whether KotlinPoet would backtick-quote the property declaration it renders for [name]. - * - * KotlinPoet escapes a declaration name for four independent reasons -- not a legal Java identifier, - * one of its own reserved `KEYWORDS`, contains `$`, or is all underscores -- and both the rule and - * the keyword set are `internal` to it, so a copy here would drift. Rendering a throwaway - * [PropertySpec] asks KotlinPoet directly instead. - * - * Tests for a backtick anywhere in the rendered text rather than for `` `$name` `` specifically: - * KotlinPoet's line wrapper substitutes a space for the characters it reserves as wrapping markers - * (U+00B7 and U+2662), so such a name is escaped in the output without appearing there verbatim. - * Callers must rule out a name containing its own backtick first -- KotlinPoet treats one as already - * escaped and skips all four checks. - */ -private fun needsKotlinPoetDeclarationBackticks(name: String): Boolean = - PropertySpec.builder(name, ANY).build().toString().contains('`') - -/** - * Formats a Kotlin property name for use as the name token in a KDoc `@property` tag. - * - * KDoc's `@property` tag takes exactly one name token before the description text begins, so a - * property name containing a space or other non-identifier character (e.g. the Kotlin property - * `` `My Col` `` generated for a quoted SQL column `"My Col"`) must be wrapped in backticks here too - * -- otherwise `@property My Col Some comment.` reads as a property literally named `My`. The name is - * left bare only when the declaration KotlinPoet renders for it is bare too, so the two never - * disagree. - * - * Uses [wrapInBacktickDelimiter]'s longest-run rule rather than a fixed single-backtick wrap, since a - * name containing its own literal backtick (e.g. `` a`b ``) would otherwise close the `@property` - * tag's span early, corrupting the rest of the line. - * - * Returns `null` — decline, emit no `@property` line at all — when [this] contains a literal - * block-comment open or close delimiter ([containsUnescapableBlockCommentDelimiter]): widening the - * backtick delimiter fixes the span, but KotlinPoet's KDoc emission still rewrites the delimiter - * itself to an HTML entity, which would render a tag naming a different property than the one - * actually declared. - * - * This fixes only the KDoc span; it does not and cannot fix the Kotlin property declaration itself - * (`` public val `a\`b`: ... ``), which is not valid Kotlin — a backtick-quoted identifier cannot - * contain a backtick, and there is no escape for one. That is a separate, pre-existing defect in how - * a column's raw database identifier becomes a Kotlin property name, left unfixed here because the - * same field also carries the identifier back into generated SQL and catalog lookups. - */ -private fun String.formatAsKdocPropertyReference(): String? = when { - !contains('`') && !needsKotlinPoetDeclarationBackticks(this) -> this - containsUnescapableBlockCommentDelimiter(this) -> null - else -> wrapInBacktickDelimiter(this) -} - -/** - * A PostgreSQL identifier that never needs double-quoting when written back into SQL: starts with - * a lowercase letter or underscore, followed by any number of lowercase letters, digits, - * underscores, or dollar signs (Postgres's own `SAFE_IDENTIFIER` rule, matching what an - * already-live-connected caller does in [JdbcAnalyzer.buildIdentifierQuoter] — this copy exists - * because [TypeRepository] has no connection of its own to query, per this file's own doc comment - * on why `TypeRepository` re-lexes rather than re-querying). Matching this pattern is necessary but - * not sufficient — [quoteSqlIdentifierIfNeeded] additionally rejects a reserved word, which this - * pattern alone cannot rule out (`order` and `user` both match it). - */ -private val SQL_UNQUOTED_IDENTIFIER = Regex("[a-z_][a-z0-9_\$]*") - -/** - * Double-quotes [identifier] exactly as PostgreSQL itself requires it to be written back into SQL - * — doubling any embedded `"` per PostgreSQL's own quoted-identifier escape rule — unless both - * [SQL_UNQUOTED_IDENTIFIER] accepts it bare AND it is not one of [reservedWords]. - * - * Without the [SQL_UNQUOTED_IDENTIFIER] half, a mixed-case or space-containing column name (`"Foo"`, - * `"My Col"`) would render bare as `table.Foo`/`table.My Col` — text that reads back as PostgreSQL - * folding `Foo` to `foo`, or as two unrelated tokens instead of one qualified reference - * (`SELECT tq.Foo FROM tq` fails with `column tq.foo does not exist`). - * - * Without the [reservedWords] half, a relation or column named after a reserved word (`order`, - * `user`) — which [SQL_UNQUOTED_IDENTIFIER] alone cannot distinguish from any other all-lowercase - * identifier — would render bare too: `` `order.id` `` reads back as `SELECT order.id FROM "order"`, - * which PostgreSQL rejects with `syntax error at or near "."`, since an unquoted `order` is parsed as - * the reserved keyword, not a table reference. [reservedWords] is always the connected server's own - * live keyword set ([JdbcAnalyzer.fetchReservedWords]), since PostgreSQL's reserved-word list drifts - * across versions. - */ -private fun quoteSqlIdentifierIfNeeded(identifier: String, reservedWords: Set): String = - if (SQL_UNQUOTED_IDENTIFIER.matches(identifier) && identifier !in reservedWords) { - identifier - } else { - "\"${identifier.replace("\"", "\"\"")}\"" - } - -/** - * Returns a source reference string for display in KDoc, or `null` if none is available (either - * there is nothing to reference, or [markdownInlineCodeSpan] could not render it faithfully — see - * that function's own KDoc for when that happens). - * - * - For columns from a table: `` `table."Column"` `` — each identifier individually quoted via - * [quoteSqlIdentifierIfNeeded] exactly as PostgreSQL requires it written back into SQL, so this - * can always be pasted into a query verbatim. - * - For computed expressions: `` `COUNT(*)` `` - * - * @param reservedWords The connected server's reserved keywords, forwarded to - * [quoteSqlIdentifierIfNeeded]. - */ -private fun PropertySource.sourceReference(reservedWords: Set): String? = when { - sourceTable != null -> { - val qualifiedColumn = sourceColumn?.let { quoteSqlIdentifierIfNeeded(it, reservedWords) }.orEmpty() - markdownInlineCodeSpan("${quoteSqlIdentifierIfNeeded(sourceTable, reservedWords)}.$qualifiedColumn") - } - expression.isNotEmpty() -> markdownInlineCodeSpan(expression) - else -> null -} - -/** - * The longest run of consecutive backtick characters anywhere in [text], or `0` if [text] contains - * none. Used by [markdownInlineCodeSpan] and [markdownFenceDelimiter] to pick a delimiter that can - * never be mistaken for a same-length run already inside [text]. - */ -private fun longestBacktickRun(text: String): Int { - var longest = 0 - var current = 0 - for (character in text) { - if (character == '`') { - current++ - if (current > longest) longest = current - } else { - current = 0 - } - } - return longest -} - -/** - * Wraps [text] in a Markdown inline code span that renders back to exactly [text], or `null` if no - * inline code span can carry it faithfully. - * - * Two hazards: - * - A run of backticks inside [text] as long as the span's own delimiter would be read as the - * closing delimiter, ending the span early. Fixed by using a delimiter one backtick longer than - * [text]'s own longest run ([longestBacktickRun]), with a padding space on each side when [text] - * itself starts or ends with a backtick. - * - A raw newline inside [text] (e.g. a string literal containing one). CommonMark folds a line - * ending inside an inline code span to a single space when rendering, silently changing the value. - * No delimiter choice can fix this — the corruption happens during rendering — so this declines. - * - A literal block-comment open or close delimiter inside [text] — see - * [containsUnescapableBlockCommentDelimiter] for why that is a third, un-fixable-by-delimiter - * hazard. - */ -internal fun markdownInlineCodeSpan(text: String): String? { - if (text.contains('\n') || text.contains('\r')) return null - if (containsUnescapableBlockCommentDelimiter(text)) return null - return wrapInBacktickDelimiter(text) -} - -/** - * Whether [text] contains `/*` or `*/`. KotlinPoet's KDoc emission unconditionally rewrites either to - * `/*`/`*/` so a literal block comment can never prematurely close the surrounding KDoc - * comment, but CommonMark never decodes that HTML entity back inside an inline code span or fenced - * code block — the two constructs [markdownInlineCodeSpan] and [TypeSpec.Builder.addClassKdoc]'s - * `sql` block render as. Once that rewrite happens there is no delimiter choice that can carry [text] - * back to its own value, so both callers decline instead. - */ -internal fun containsUnescapableBlockCommentDelimiter(text: String): Boolean = - text.contains("/*") || text.contains("*/") - -/** - * Backslash-escapes every literal backtick in [text] so it can never be read as a CommonMark - * inline-code-span delimiter, without altering the character [text] renders as. - * - * [TypeSpec.Builder.addClassKdoc] appends every property's [PropertySource.comment] into one - * continuous CommonMark paragraph shared by every `@property` line, so an unescaped backtick in one - * comment could pair with a backtick belonging to a later property's own source-reference span, - * corrupting every span in between. Escaping here removes the character from delimiter-matching - * entirely. - * - * Escapes [text]'s own literal backslashes first, before escaping backticks: escaping only the - * backtick is defeated when [text] already has a backslash immediately before one (e.g. - * `` 'weird \`' ``) — the naive replacement produces `` \\` ``, which CommonMark reads as an escaped - * backslash followed by an unescaped, still-open backtick. - */ -internal fun escapeMarkdownBacktick(text: String): String = text.replace("\\", "\\\\").replace("`", "\\`") - -/** - * Wraps [text] in a backtick-delimited span using a delimiter one backtick longer than [text]'s own - * longest internal run ([longestBacktickRun]), with a padding space on each side when [text] starts - * or ends with a backtick. Shared by [markdownInlineCodeSpan] and [formatAsKdocPropertyReference], - * whose spans are both subject to the same backtick-collision hazard. - */ -private fun wrapInBacktickDelimiter(text: String): String { - val delimiter = "`".repeat(longestBacktickRun(text) + 1) - val needsPadding = text.startsWith("`") || text.endsWith("`") - return if (needsPadding) "$delimiter $text $delimiter" else "$delimiter$text$delimiter" -} - -/** - * A backtick-fence delimiter (` ``` `, or longer) that can open a Markdown fenced code block - * containing [text] without [text] itself supplying a same-length or longer backtick run that - * CommonMark would read as the block's own closing fence — one backtick longer than [text]'s own - * longest run ([longestBacktickRun]), never shorter than the conventional 3. - */ -internal fun markdownFenceDelimiter(text: String): String = "`".repeat(maxOf(3, longestBacktickRun(text) + 1)) - -/** - * Collapses the cosmetic whitespace [stripComments]' own single-space substitution can leave behind - * in an expression about to be embedded verbatim in generated KDoc — a comment directly after an - * opening parenthesis or before a closing one (`UPPER(/* x */a)` strips to `UPPER( a)`) reads oddly - * there, even though that space is exactly right for [stripComments]' own purpose of never fusing two - * tokens a comment used to separate. Applied only where [resolveNodeTreeProvenanceExpression] returns - * an expression for KDoc, never inside [stripComments] itself. - * - * Collapses whitespace outside a single-quoted literal, a dollar-quoted string, a quoted identifier, - * or a comment to a single space, then removes a single such space immediately after `(` or before - * `)` — never semantically significant in SQL, so this can never change what the expression means. - * - * Walks every span verbatim via [skipLexicalToken], the same primitive [stripComments] uses, rather - * than a second, independently-written scanner: a plain whitespace-collapse regex can't tell a - * cosmetic space from one inside the developer's own SQL, and would rewrite a quoted identifier's - * internal spacing (`"My Col"` to `"My Col"`, a column name PostgreSQL then rejects) or a string - * literal's contents (`'( x )'` to `'(x)'`) instead of merely the padding around it. - */ -internal fun collapseCosmeticWhitespace(text: String): String { - val trimmed = text.trim() - val builder = StringBuilder(trimmed.length) - var i = 0 - while (i < trimmed.length) { - val afterToken = skipLexicalToken(trimmed, i) - if (afterToken != i) { - builder.append(trimmed, i, afterToken) - i = afterToken - continue - } - val character = trimmed[i] - if (!character.isWhitespace()) { - builder.append(character) - i++ - continue - } - var afterWhitespace = i - while (afterWhitespace < trimmed.length && trimmed[afterWhitespace].isWhitespace()) afterWhitespace++ - val precededByOpenParenthesis = builder.isNotEmpty() && builder.last() == '(' - val followedByCloseParenthesis = afterWhitespace < trimmed.length && trimmed[afterWhitespace] == ')' - if (!precededByOpenParenthesis && !followedByCloseParenthesis) builder.append(' ') - i = afterWhitespace - } - return builder.toString() + POSTGRES_BASE_TYPES[typeName.removePrefix("pg_catalog.")]?.let { ScalarSqlMappable(it.codec, notNull) } } diff --git a/generator/src/test/kotlin/norm/generator/ColumnTypeMappingTest.kt b/generator/src/test/kotlin/norm/generator/ColumnTypeMappingTest.kt index 39bbae28..7dd0b782 100644 --- a/generator/src/test/kotlin/norm/generator/ColumnTypeMappingTest.kt +++ b/generator/src/test/kotlin/norm/generator/ColumnTypeMappingTest.kt @@ -835,7 +835,8 @@ class ColumnTypeMappingTest { @Test fun `non-null timestamptz array element read is null-safe`() { // A NOT NULL timestamptz[] column can still contain NULL elements, so the element read must - // use the nullable InstantSqlMappable form. The non-null form would NPE on a NULL element. + // use the nullable InstantViaOffsetDateTimeCodec form. The non-null form would NPE on a NULL + // element. val col = column("moments", type = "timestamptz", isArray = true, notNull = true) val accessor = typeRepository.resolveMappableType(col).resultSetAction(1) assertThat(accessor.toString()) @@ -981,20 +982,19 @@ class ColumnTypeMappingTest { ) /** - * Anti-drift sweep for [postgresArrayElementTypeName], the same intent as - * [DomainBaseTypeAntiDriftSweep] but pinned against a hardcoded, independently-verified - * classification rather than [BASE_TYPE_RESOLVERS] membership. A membership check is a - * tautology here: every [BASE_TYPE_RESOLVERS] key that is not folded still passes itself - * through unchanged (`postgresArrayElementTypeName`'s `else` branch), and every SQL-spelling - * alias is, by construction, also a [BASE_TYPE_RESOLVERS] key — so deleting every fold branch - * would still leave every folded result a [BASE_TYPE_RESOLVERS] key and a membership check - * green. + * Anti-drift sweep for [postgresArrayElementTypeName], pinned against a hardcoded, + * independently-verified classification rather than [POSTGRES_BASE_TYPES] membership. A + * membership check is a tautology here: every [POSTGRES_BASE_TYPES] key that is not folded + * still passes itself through unchanged (`postgresArrayElementTypeName`'s `else` branch), and + * every SQL-spelling alias is, by construction, also a [POSTGRES_BASE_TYPES] key — so deleting + * every fold branch would still leave every folded result a [POSTGRES_BASE_TYPES] key and a + * membership check green. * * [expectedCanonicalNameByAlias] and [alreadyCanonicalNames] below come from a PostgreSQL 17 * server via `SELECT typname FROM pg_type WHERE oid = to_regtype(?)` — see - * [postgresArrayElementTypeName]'s KDoc — and never derived from [BASE_TYPE_RESOLVERS] or + * [postgresArrayElementTypeName]'s KDoc — and never derived from [POSTGRES_BASE_TYPES] or * [postgresArrayElementTypeName] themselves. The set-equality assertion catches a new - * [BASE_TYPE_RESOLVERS] key added without being classified into either bucket; the per-alias + * [POSTGRES_BASE_TYPES] key added without being classified into either bucket; the per-alias * assertions catch a fold branch that is deleted, or wrong, by checking the actual fold result * against this table's fixed expectation rather than a self-referential set. * @@ -1022,7 +1022,7 @@ class ColumnTypeMappingTest { "text", "varchar", "bpchar", "uuid", ) - assertThat(BASE_TYPE_RESOLVERS.keys - serialVariants) + assertThat(POSTGRES_BASE_TYPES.keys - serialVariants) .isEqualTo(expectedCanonicalNameByAlias.keys + alreadyCanonicalNames) expectedCanonicalNameByAlias.forEach { (alias, expectedCanonical) -> @@ -1500,7 +1500,7 @@ class ColumnTypeMappingTest { /** * Tests for domain base types beyond TEXT and INTEGER. * - * Each base type exercises both [resolveJdbcTypeInfo] (JDBC method metadata) and + * Each base type exercises both [resolveWireCodec] (JDBC method metadata) and * [domainKotlinBaseType] (Kotlin type mapping). These two functions must stay in sync — * a type supported in one but not the other is a bug. */ @@ -1745,10 +1745,10 @@ class ColumnTypeMappingTest { /** * Regression coverage for the build-breaking bug: `CREATE DOMAIN d AS timestamptz` (or `uuid`, * `date`, `time`, `timetz`, `bytea`, `oid`) aborted code generation entirely, because - * [resolveJdbcTypeInfo] had no entry for any of these types even though + * [resolveWireCodec] had no entry for any of these types even though * [TypeRepository.resolveBaseType] supports every one of them as a plain column type. Each read * assertion below is verified against pgjdbc 42.7.13's actual `PgResultSet`/`PgPreparedStatement` - * source (see [resolveJdbcTypeInfo]'s KDoc for the specific methods checked), not assumed. + * source (see [resolveWireCodec]'s KDoc for the specific methods checked), not assumed. */ @Nested inner class DomainOverJavaTimeAndOtherNonPrimitiveBaseTypes { @@ -2050,183 +2050,223 @@ class ColumnTypeMappingTest { } /** - * Direct tests for [resolveJdbcTypeInfo], verifying the JDBC method metadata - * for each supported Postgres base type. + * Direct tests for [resolveWireCodec], verifying the rendered JDBC read/write shape for each + * supported Postgres base type. * - * These tests ensure that the getter/setter names, primitivity flags, and SQL type constants - * are correct for each base type. A mistake here would generate code that compiles but uses - * the wrong JDBC method at runtime. + * These tests ensure that the getter/setter calls and `Types` constants are correct for each + * base type. A mistake here would generate code that compiles but uses the wrong JDBC method at + * runtime. */ @Nested inner class DomainBaseTypeResolution { @Test fun `text resolves to getString and setString`() { - val info = resolveJdbcTypeInfo("text")!! - assertThat(info.getterName).isEqualTo("getString") - assertThat(info.setterName).isEqualTo("setString") - assertThat(info.isPrimitive).isFalse() - assertThat(info.sqlTypeConstant).isEqualTo("VARCHAR") + val codec = resolveWireCodec("text")!! + assertThat(codec.read(1, false).toString()).isEqualTo("getString(1)") + assertThat(codec.read(1, true).toString()).isEqualTo("getString(1)") + assertThat(codec.write(1, CodeBlock.of("value")).toString()).isEqualTo("setString(1, value)") + assertThat(codec.writeNullable(1, CodeBlock.of("value")).toString()).isEqualTo("setString(1, value)") + assertThat(codec.writeNull(1).toString()).isEqualTo("setNull(1, java.sql.Types.VARCHAR)") } @Test fun `varchar resolves same as text`() { - val info = resolveJdbcTypeInfo("varchar")!! - assertThat(info.getterName).isEqualTo("getString") - assertThat(info.setterName).isEqualTo("setString") - assertThat(info.isPrimitive).isFalse() - assertThat(info.sqlTypeConstant).isEqualTo("VARCHAR") + val codec = resolveWireCodec("varchar")!! + assertThat(codec.write(1, CodeBlock.of("value")).toString()).isEqualTo("setString(1, value)") + assertThat(codec.writeNull(1).toString()).isEqualTo("setNull(1, java.sql.Types.VARCHAR)") } @Test fun `bpchar resolves same as text`() { - val info = resolveJdbcTypeInfo("bpchar")!! - assertThat(info.getterName).isEqualTo("getString") - assertThat(info.sqlTypeConstant).isEqualTo("VARCHAR") + val codec = resolveWireCodec("bpchar")!! + assertThat(codec.read(1, false).toString()).isEqualTo("getString(1)") + assertThat(codec.writeNull(1).toString()).isEqualTo("setNull(1, java.sql.Types.VARCHAR)") } @Test fun `int2 resolves to getShort and setShort`() { - val info = resolveJdbcTypeInfo("int2")!! - assertThat(info.getterName).isEqualTo("getShort") - assertThat(info.setterName).isEqualTo("setShort") - assertThat(info.isPrimitive).isTrue() - assertThat(info.sqlTypeConstant).isEqualTo("SMALLINT") + val codec = resolveWireCodec("int2")!! + assertThat(codec.read(1, false).toString()).isEqualTo("getShort(1)") + assertThat(codec.read(1, true).toString()).isEqualTo("getShort(1).takeUnless { wasNull() }") + assertThat(codec.write(1, CodeBlock.of("value")).toString()).isEqualTo("setShort(1, value)") + assertThat(codec.writeNullable(1, CodeBlock.of("value")).toString()).isEqualTo("norm.setShort(1, value)") + assertThat(codec.writeNull(1).toString()).isEqualTo("setNull(1, java.sql.Types.SMALLINT)") } @Test fun `int4 resolves to getInt and setInt`() { - val info = resolveJdbcTypeInfo("int4")!! - assertThat(info.getterName).isEqualTo("getInt") - assertThat(info.setterName).isEqualTo("setInt") - assertThat(info.isPrimitive).isTrue() - assertThat(info.sqlTypeConstant).isEqualTo("INTEGER") + val codec = resolveWireCodec("int4")!! + assertThat(codec.read(1, false).toString()).isEqualTo("getInt(1)") + assertThat(codec.read(1, true).toString()).isEqualTo("getInt(1).takeUnless { wasNull() }") + assertThat(codec.write(1, CodeBlock.of("value")).toString()).isEqualTo("setInt(1, value)") + assertThat(codec.writeNullable(1, CodeBlock.of("value")).toString()).isEqualTo("norm.setInt(1, value)") + assertThat(codec.writeNull(1).toString()).isEqualTo("setNull(1, java.sql.Types.INTEGER)") } @Test fun `int8 resolves to getLong and setLong`() { - val info = resolveJdbcTypeInfo("int8")!! - assertThat(info.getterName).isEqualTo("getLong") - assertThat(info.setterName).isEqualTo("setLong") - assertThat(info.isPrimitive).isTrue() - assertThat(info.sqlTypeConstant).isEqualTo("BIGINT") + val codec = resolveWireCodec("int8")!! + assertThat(codec.read(1, false).toString()).isEqualTo("getLong(1)") + assertThat(codec.read(1, true).toString()).isEqualTo("getLong(1).takeUnless { wasNull() }") + assertThat(codec.write(1, CodeBlock.of("value")).toString()).isEqualTo("setLong(1, value)") + assertThat(codec.writeNullable(1, CodeBlock.of("value")).toString()).isEqualTo("norm.setLong(1, value)") + assertThat(codec.writeNull(1).toString()).isEqualTo("setNull(1, java.sql.Types.BIGINT)") } @Test fun `float4 resolves to getFloat and setFloat`() { - val info = resolveJdbcTypeInfo("float4")!! - assertThat(info.getterName).isEqualTo("getFloat") - assertThat(info.setterName).isEqualTo("setFloat") - assertThat(info.isPrimitive).isTrue() - assertThat(info.sqlTypeConstant).isEqualTo("REAL") + val codec = resolveWireCodec("float4")!! + assertThat(codec.read(1, false).toString()).isEqualTo("getFloat(1)") + assertThat(codec.read(1, true).toString()).isEqualTo("getFloat(1).takeUnless { wasNull() }") + assertThat(codec.write(1, CodeBlock.of("value")).toString()).isEqualTo("setFloat(1, value)") + assertThat(codec.writeNullable(1, CodeBlock.of("value")).toString()).isEqualTo("norm.setFloat(1, value)") + assertThat(codec.writeNull(1).toString()).isEqualTo("setNull(1, java.sql.Types.REAL)") } @Test fun `float8 resolves to getDouble and setDouble`() { - val info = resolveJdbcTypeInfo("float8")!! - assertThat(info.getterName).isEqualTo("getDouble") - assertThat(info.setterName).isEqualTo("setDouble") - assertThat(info.isPrimitive).isTrue() - assertThat(info.sqlTypeConstant).isEqualTo("DOUBLE") + val codec = resolveWireCodec("float8")!! + assertThat(codec.read(1, false).toString()).isEqualTo("getDouble(1)") + assertThat(codec.read(1, true).toString()).isEqualTo("getDouble(1).takeUnless { wasNull() }") + assertThat(codec.write(1, CodeBlock.of("value")).toString()).isEqualTo("setDouble(1, value)") + assertThat(codec.writeNullable(1, CodeBlock.of("value")).toString()).isEqualTo("norm.setDouble(1, value)") + assertThat(codec.writeNull(1).toString()).isEqualTo("setNull(1, java.sql.Types.DOUBLE)") } @Test fun `bool resolves to getBoolean and setBoolean`() { - val info = resolveJdbcTypeInfo("bool")!! - assertThat(info.getterName).isEqualTo("getBoolean") - assertThat(info.setterName).isEqualTo("setBoolean") - assertThat(info.isPrimitive).isTrue() - assertThat(info.sqlTypeConstant).isEqualTo("BOOLEAN") + val codec = resolveWireCodec("bool")!! + assertThat(codec.read(1, false).toString()).isEqualTo("getBoolean(1)") + assertThat(codec.read(1, true).toString()).isEqualTo("getBoolean(1).takeUnless { wasNull() }") + assertThat(codec.write(1, CodeBlock.of("value")).toString()).isEqualTo("setBoolean(1, value)") + assertThat(codec.writeNullable(1, CodeBlock.of("value")).toString()).isEqualTo("norm.setBoolean(1, value)") + assertThat(codec.writeNull(1).toString()).isEqualTo("setNull(1, java.sql.Types.BOOLEAN)") } @Test fun `numeric resolves to getBigDecimal and setBigDecimal`() { - val info = resolveJdbcTypeInfo("numeric")!! - assertThat(info.getterName).isEqualTo("getBigDecimal") - assertThat(info.setterName).isEqualTo("setBigDecimal") - assertThat(info.isPrimitive).isFalse() - assertThat(info.sqlTypeConstant).isEqualTo("NUMERIC") + val codec = resolveWireCodec("numeric")!! + assertThat(codec.read(1, false).toString()).isEqualTo("getBigDecimal(1)") + assertThat(codec.read(1, true).toString()).isEqualTo("getBigDecimal(1)") + assertThat(codec.write(1, CodeBlock.of("value")).toString()).isEqualTo("setBigDecimal(1, value)") + assertThat(codec.writeNullable(1, CodeBlock.of("value")).toString()).isEqualTo("setBigDecimal(1, value)") + assertThat(codec.writeNull(1).toString()).isEqualTo("setNull(1, java.sql.Types.NUMERIC)") } @Test fun `jsonb resolves to getString and setObject with Types OTHER`() { - val info = resolveJdbcTypeInfo("jsonb")!! - assertThat(info.getterName).isEqualTo("getString") + val codec = resolveWireCodec("jsonb")!! + assertThat(codec.read(1, false).toString()).isEqualTo("getString(1)") // setObject(..., Types.OTHER) is required — Postgres JDBC rejects setString() for jsonb columns - assertThat(info.setterName).isEqualTo("setObject") - assertThat(info.isPrimitive).isFalse() - assertThat(info.sqlTypeConstant).isEqualTo("OTHER") - assertThat(info.useSqlTypeHint).isTrue() + assertThat(codec.write(1, CodeBlock.of("value")).toString()) + .isEqualTo("setObject(1, value, java.sql.Types.OTHER)") + assertThat(codec.writeNullable(1, CodeBlock.of("value")).toString()) + .isEqualTo("setObject(1, value, java.sql.Types.OTHER)") + assertThat(codec.writeNull(1).toString()).isEqualTo("setNull(1, java.sql.Types.OTHER)") } @Test fun `json resolves to getString and setObject with Types OTHER`() { - val info = resolveJdbcTypeInfo("json")!! - assertThat(info.getterName).isEqualTo("getString") - assertThat(info.setterName).isEqualTo("setObject") - assertThat(info.isPrimitive).isFalse() - assertThat(info.sqlTypeConstant).isEqualTo("OTHER") - assertThat(info.useSqlTypeHint).isTrue() + val codec = resolveWireCodec("json")!! + assertThat(codec.read(1, false).toString()).isEqualTo("getString(1)") + assertThat(codec.write(1, CodeBlock.of("value")).toString()) + .isEqualTo("setObject(1, value, java.sql.Types.OTHER)") + assertThat(codec.writeNull(1).toString()).isEqualTo("setNull(1, java.sql.Types.OTHER)") } @Test fun `uuid resolves to getObject and setObject with a UUID class hint`() { - val info = resolveJdbcTypeInfo("uuid")!! - assertThat(info.getterName).isEqualTo("getObject") - assertThat(info.setterName).isEqualTo("setObject") - assertThat(info.isPrimitive).isFalse() - assertThat(info.sqlTypeConstant).isEqualTo("OTHER") - assertThat(info.getterClassHint).isEqualTo(ClassName("java.util", "UUID")) + val codec = resolveWireCodec("uuid")!! + assertThat(codec.read(1, false).toString()).isEqualTo("getObject(1, java.util.UUID::class.java)") + assertThat(codec.read(1, true).toString()).isEqualTo("getObject(1, java.util.UUID::class.java)") + assertThat(codec.write(1, CodeBlock.of("value")).toString()).isEqualTo("setObject(1, value)") + assertThat(codec.writeNullable(1, CodeBlock.of("value")).toString()).isEqualTo("setObject(1, value)") + assertThat(codec.writeNull(1).toString()).isEqualTo("setNull(1, java.sql.Types.OTHER)") } @Test fun `unsupported type returns null`() { // xml has no entry anywhere -- Norm has never mapped it to a Kotlin type, as a plain column - // type or a domain base. bytea is supported (see BASE_TYPE_RESOLVERS/DomainBaseTypes below) -- - // it used to return null here, which is exactly the bug this fix closes: CREATE DOMAIN d AS - // bytea aborted code generation entirely. - assertThat(resolveJdbcTypeInfo("xml")).isEqualTo(null) + // type or a domain base. bytea is supported (see POSTGRES_BASE_TYPES) -- it used to return + // null here, which is exactly the bug this fix closes: CREATE DOMAIN d AS bytea aborted code + // generation entirely. + assertThat(resolveWireCodec("xml")).isEqualTo(null) } } /** - * Anti-drift sweep for the bug where a domain over a common base type (e.g. `CREATE DOMAIN d AS - * timestamptz`) aborted code generation: [resolveJdbcTypeInfo] must have an entry for every - * canonical type name [BASE_TYPE_RESOLVERS] accepts, since [TypeRepository]'s domain resolution - * chains through [resolveJdbcTypeInfo] for the domain's base type. The corpus is - * [BASE_TYPE_RESOLVERS]'s own keys -- the exact set [TypeRepository.resolveBaseType] accepts -- - * rather than a hand-copied list here that could independently drift from it. + * Regression coverage for the build-breaking bug: the `uuid` entry in [resolveWireCodec] used + * the generic `"getObject"` getter with no class hint, which generates a bare `getObject(index)` + * read. `java.sql.ResultSet.getObject(int)` is declared to return `Object`, so that read is + * statically `Any` in Kotlin no matter what concrete type pgjdbc's `PgResultSet` returns at + * runtime — it does not compile as an argument to `ColumnAdapter.decode`. + * This sweeps every entry [resolveWireCodec] can produce, for both a `NOT NULL` and a nullable + * read, so a future type added with the same mistake fails immediately, rather than surfacing + * only when a generated scenario happens to be compiled. */ @Nested - inner class DomainBaseTypeAntiDriftSweep { + inner class GetObjectReadsRequireAClassHint { @Test - fun `every canonical base type resolveBaseType accepts has a resolveJdbcTypeInfo entry`() { - val unsupported = BASE_TYPE_RESOLVERS.keys.filter { resolveJdbcTypeInfo(it) == null } - assertThat(unsupported).isEmpty() + fun `every resolveWireCodec entry's read avoids a bare getObject`() { + val bareGetObjectReads = POSTGRES_BASE_TYPES.keys + .flatMap { key -> + val codec = resolveWireCodec(key)!! + listOf(codec.read(1, false).toString(), codec.read(1, true).toString()) + } + .filter { Regex("""\bgetObject\(1\)""").containsMatchIn(it) } + + assertThat(bareGetObjectReads).isEmpty() } } /** - * Regression coverage for the build-breaking bug: the `uuid` entry in [resolveJdbcTypeInfo] used - * the generic `"getObject"` getter with no [JdbcTypeInfo.getterClassHint], which generates a bare - * `getObject(index)` read. `java.sql.ResultSet.getObject(int)` is declared to return `Object`, so - * that read is statically `Any` in Kotlin no matter what concrete type pgjdbc's `PgResultSet` - * returns at runtime — it does not compile as an argument to `ColumnAdapter.decode`. This sweeps every entry [resolveJdbcTypeInfo] can produce so a future type added - * with the same mistake fails immediately, rather than surfacing only when a generated scenario - * happens to be compiled. + * Pins Correction 1 directly: [ScalarSqlMappable.statementAction] for a nullable plain column + * must keep rendering the exact shape each of the five [WireCodec] kinds rendered before the + * [WireCodec] refactor. Golden-file invariance alone would not catch a regression here — a + * scenario compiling successfully says nothing about which exact `wasNull()`/`setNull` fallback + * ran, only that some shape did. */ @Nested - inner class GetObjectReadsRequireAClassHint { + inner class NullablePlainColumnWriteShapes { + + @Test + fun `PrimitiveCodec nullable write uses the norm set extension`() { + val codec = resolveWireCodec("int4")!! + val action = ScalarSqlMappable(codec, notNull = false).statementAction(1, CodeBlock.of("value")) + assertThat(action.toString()).isEqualTo("norm.setInt(1, value)") + } + + @Test + fun `ObjectGetterCodec nullable write uses the plain setter`() { + val codec = resolveWireCodec("text")!! + val action = ScalarSqlMappable(codec, notNull = false).statementAction(1, CodeBlock.of("value")) + assertThat(action.toString()).isEqualTo("setString(1, value)") + } + + @Test + fun `ClassHintedObjectCodec nullable write uses the plain setObject`() { + val codec = resolveWireCodec("uuid")!! + val action = ScalarSqlMappable(codec, notNull = false).statementAction(1, CodeBlock.of("value")) + assertThat(action.toString()).isEqualTo("setObject(1, value)") + } @Test - fun `every resolveJdbcTypeInfo entry using the generic getObject getter supplies a class hint`() { - val entriesMissingAClassHint = BASE_TYPE_RESOLVERS.keys - .mapNotNull { resolveJdbcTypeInfo(it) } - .filter { it.getterName == "getObject" && it.getterClassHint == null } + fun `TypesOtherCodec nullable write uses setObject with Types OTHER`() { + val codec = resolveWireCodec("jsonb")!! + val action = ScalarSqlMappable(codec, notNull = false).statementAction(1, CodeBlock.of("value")) + assertThat(action.toString()).isEqualTo("setObject(1, value, java.sql.Types.OTHER)") + } - assertThat(entriesMissingAClassHint).isEmpty() + @Test + fun `InstantViaOffsetDateTimeCodec nullable write uses the safe-call setNull fallback`() { + val codec = resolveWireCodec("timestamptz")!! + val action = ScalarSqlMappable(codec, notNull = false).statementAction(1, CodeBlock.of("value")) + assertThat(action.toString()).isEqualTo( + "value?.let { setObject(1, java.time.OffsetDateTime.ofInstant(it, java.time.ZoneOffset.UTC)) } " + + "?: setNull(1, java.sql.Types.TIMESTAMP_WITH_TIMEZONE)", + ) } } diff --git a/generator/src/test/kotlin/norm/generator/CrudQuerySynthesizerTest.kt b/generator/src/test/kotlin/norm/generator/CrudQuerySynthesizerTest.kt index fae0220a..282b088d 100644 --- a/generator/src/test/kotlin/norm/generator/CrudQuerySynthesizerTest.kt +++ b/generator/src/test/kotlin/norm/generator/CrudQuerySynthesizerTest.kt @@ -157,7 +157,6 @@ class CrudQuerySynthesizerTest { ), ) val catalog = Catalog( - defaultSchema = "analytics", schemas = listOf(Schema(name = "analytics", tables = listOf(table))), ) @@ -276,7 +275,6 @@ class CrudQuerySynthesizerTest { ), ) val catalog = Catalog( - defaultSchema = "select", schemas = listOf(Schema(name = "select", tables = listOf(table))), ) val quoter = quoteOnly("select", "order") @@ -376,9 +374,9 @@ class CrudQuerySynthesizerTest { column("My Col", "text"), ) val catalog = catalog(table) - // Mirrors JdbcAnalyzer.buildIdentifierQuoter's real policy (needsQuoting: any character - // outside "[a-z_][a-z0-9_$]*", which "Foo"'s uppercase "F" and "My Col"'s space both are) - // without needing a live connection just to fetch the (here, irrelevant) reserved-word set. + // Mirrors quoteSqlIdentifierIfNeeded's real policy (quotes any character outside + // "[a-z_][a-z0-9_$]*", which "Foo"'s uppercase "F" and "My Col"'s space both are) without + // needing a live connection just to fetch the (here, irrelevant) reserved-word set. val quoter = { identifier: String -> if (identifier.matches(Regex("[a-z_][a-z0-9_\$]*"))) identifier else "\"$identifier\"" } @@ -449,7 +447,6 @@ class CrudQuerySynthesizerTest { ) private fun catalog(vararg tables: Table) = Catalog( - defaultSchema = "public", schemas = listOf(Schema(name = "public", tables = tables.toList())), ) } diff --git a/generator/src/test/kotlin/norm/generator/DomainBuilderTest.kt b/generator/src/test/kotlin/norm/generator/DomainBuilderTest.kt index 0ae7212e..50099b6e 100644 --- a/generator/src/test/kotlin/norm/generator/DomainBuilderTest.kt +++ b/generator/src/test/kotlin/norm/generator/DomainBuilderTest.kt @@ -122,7 +122,7 @@ class DomainBuilderTest { @Test fun `json domain generates a String value class`() { // Postgres accepts CREATE DOMAIN d AS json, so json must be usable as a domain base. The - // Types.OTHER binding lives in the JdbcTypeInfo, not in the wrapped Kotlin type. + // Types.OTHER binding lives in the WireCodec, not in the wrapped Kotlin type. val domain = Domain(name = "json_doc", baseType = "json", comment = "") val output = generateValueClassCode(domain, "example") assertThat(output).contains("import kotlin.String") @@ -139,7 +139,7 @@ class DomainBuilderTest { @Test fun `TIMESTAMPTZ domain generates an Instant value class`() { // Regression coverage for the build-breaking bug: CREATE DOMAIN d AS timestamptz used to - // abort code generation entirely -- resolveJdbcTypeInfo had no entry for timestamptz even + // abort code generation entirely -- resolveWireCodec had no entry for timestamptz even // though it is one of the most common domain base types. val domain = Domain(name = "occurred_at", baseType = "timestamptz", comment = "") val output = generateValueClassCode(domain, "example") diff --git a/generator/src/test/kotlin/norm/generator/ExplainAnalysisTest.kt b/generator/src/test/kotlin/norm/generator/ExplainAnalysisTest.kt index 13e4cda6..ecb6bb9d 100644 --- a/generator/src/test/kotlin/norm/generator/ExplainAnalysisTest.kt +++ b/generator/src/test/kotlin/norm/generator/ExplainAnalysisTest.kt @@ -17,7 +17,7 @@ import java.util.concurrent.atomic.AtomicInteger /** * Stage 2 of the `prosqlbody` cutover: `EXPLAIN (FORMAT JSON)` never executes the statement it * plans, and reports a `MERGE`'s per-relation match-optionality — the one thing - * [PgCatalogLoader.mergeAbsentVarnos]'s KDoc documents as invisible to `:varnullingrels` on the + * [ColumnNullabilityAnalyzer.mergeAbsentVarnos]'s KDoc documents as invisible to `:varnullingrels` on the * `CREATE VIEW`/`ev_action` or `prosqlbody` route. */ @Testcontainers diff --git a/generator/src/test/kotlin/norm/generator/FrameworkAnnotationTest.kt b/generator/src/test/kotlin/norm/generator/FrameworkAnnotationTest.kt index 5ad42ab9..ed559637 100644 --- a/generator/src/test/kotlin/norm/generator/FrameworkAnnotationTest.kt +++ b/generator/src/test/kotlin/norm/generator/FrameworkAnnotationTest.kt @@ -42,7 +42,6 @@ class FrameworkAnnotationTest { * Creates a catalog with the given tables in the public schema. */ private fun createCatalog(vararg tables: Table): Catalog = Catalog( - defaultSchema = "public", schemas = listOf( Schema( name = "public", diff --git a/generator/src/test/kotlin/norm/generator/GenerateCodeTest.kt b/generator/src/test/kotlin/norm/generator/GenerateCodeTest.kt index 46590de7..6788b18b 100644 --- a/generator/src/test/kotlin/norm/generator/GenerateCodeTest.kt +++ b/generator/src/test/kotlin/norm/generator/GenerateCodeTest.kt @@ -248,6 +248,61 @@ class GenerateCodeTest { assertThat(implementationFile.contents).contains("CustomJson") } + /** + * `adapterParameters` must run after `generateQueryInterface`, since a query parameter's column + * type is only resolved while building interface methods, not while constructing `SqlStatement`. + * An `UPDATE` with no result columns whose only reference to the enum is a `WHERE` parameter is the + * only shape that can catch a regression that computes `adapterParameters` too early. + */ + @Test + fun `enum referenced only as an exec query parameter still gets a constructor adapter with its default`() { + connection.createStatement().use { + it.execute( + """ + DEALLOCATE ALL; + DROP SCHEMA public CASCADE; + CREATE SCHEMA public; + GRANT ALL ON SCHEMA public TO public; + """.trimIndent(), + ) + } + + connection.createStatement().use { + it.execute( + """ + CREATE TYPE mood AS ENUM ('sad', 'ok', 'happy'); + CREATE TABLE person ( + id integer PRIMARY KEY, + name text NOT NULL, + current_mood mood NOT NULL + ); + """.trimIndent(), + ) + } + + val analyzer = JdbcAnalyzer(connection) + val catalog = analyzer.buildCatalog() + + val parsedQueries = QueryFileParser.parse( + """ + -- name: updateName :exec + UPDATE person SET name = ? WHERE current_mood = ?; + """.trimIndent(), + ) + val analyzedQueries = parsedQueries.map { analyzer.analyzeQuery(it, catalog) } + + val result = generateCode( + catalog, + analyzedQueries, + "example", + emptySet(), + analyzer.fetchReservedWords(), + ) + + val implementationFile = result.first { it.name.endsWith("PostgresQueries.kt") } + assertThat(implementationFile.contents).contains("moodAdapter: ColumnAdapter = MoodAdapter()") + } + companion object { // Embed scenarios use sqlc.embed() which is not yet supported by the JDBC analyzer private val EMBED_SCENARIOS = diff --git a/generator/src/test/kotlin/norm/generator/GroupRteSubstitutionTest.kt b/generator/src/test/kotlin/norm/generator/GroupRteSubstitutionTest.kt index b905cb7b..1f98518d 100644 --- a/generator/src/test/kotlin/norm/generator/GroupRteSubstitutionTest.kt +++ b/generator/src/test/kotlin/norm/generator/GroupRteSubstitutionTest.kt @@ -59,9 +59,9 @@ class GroupRteSubstitutionTest { @Test fun `a Var whose resolved group expression is Unknown is left unchanged`() { // An Unknown resolution means the groupexprs entry is either a parse failure or an unmodelled - // node type. Substituting it in would replace a Var that PgNodeTreeParser.parseGroupRteMap's - // coarser, Var-only fallback might still be able to resolve — so this Var must be left exactly - // as parsed, not swapped for Unknown. + // node type. Substituting it in would replace a Var that groupRteMap's coarser, Var-only + // fallback might still be able to resolve — so this Var must be left exactly as parsed, not + // swapped for Unknown. val groupRteVar = PgNodeExpression.Var(varno = 2, varattno = 1, nullingRelations = emptySet()) val groupExpressionsByVarno = mapOf(2 to listOf(PgNodeExpression.Unknown("XMLTABLE"))) @@ -122,4 +122,24 @@ class GroupRteSubstitutionTest { assertThat(result).isSameInstanceAs(const) assertThat((result as PgNodeExpression.Const).isNull).isFalse() } + + @Test + fun `a Var buried inside a JsonConstructorExpr's function is substituted`() { + // JSON_OBJECTAGG/JSON_ARRAYAGG put the underlying aggregate in `function`, not `arguments` (see + // JsonConstructorExpr.function's KDoc) — this is the one child mapChildren now reaches that the + // hand-written `when` this function used to have never touched. + val groupRteVar = PgNodeExpression.Var(varno = 2, varattno = 1, nullingRelations = emptySet()) + val resolvedVar = PgNodeExpression.Var(varno = 1, varattno = 2, nullingRelations = emptySet()) + val windowFunc = PgNodeExpression.WindowFunc(windowFunctionOid = 3125, arguments = listOf(groupRteVar)) + val jsonArrayagg = PgNodeExpression.JsonConstructorExpr( + type = PgNodeExpression.JSON_CONSTRUCTOR_TYPE_ARRAYAGG, + arguments = emptyList(), + function = windowFunc, + ) + val groupExpressionsByVarno = mapOf(2 to listOf(resolvedVar)) + + val result = substituteGroupRteVars(jsonArrayagg, groupExpressionsByVarno) as PgNodeExpression.JsonConstructorExpr + + assertThat((result.function as PgNodeExpression.WindowFunc).arguments.single()).isEqualTo(resolvedVar) + } } diff --git a/generator/src/test/kotlin/norm/generator/JdbcAnalyzerTest.kt b/generator/src/test/kotlin/norm/generator/JdbcAnalyzerTest.kt index eabcd698..9b229e12 100644 --- a/generator/src/test/kotlin/norm/generator/JdbcAnalyzerTest.kt +++ b/generator/src/test/kotlin/norm/generator/JdbcAnalyzerTest.kt @@ -39,7 +39,6 @@ class JdbcAnalyzerTest { fun `buildCatalog discovers tables`() { val catalog = analyzer.buildCatalog() - assertThat(catalog.defaultSchema).isEqualTo("public") assertThat(catalog.schemas).hasSize(1) val tables = catalog.schemas.first().tables @@ -253,6 +252,27 @@ class JdbcAnalyzerTest { assertThat(query.params[1].column!!.type.name).isEqualTo("text") } + /** + * Schema qualification (`public.`) defeats `CALL_PROCEDURE_NAME`'s bare-identifier regex, so + * `analyzeCallParameters` falls back to `prepareStatement(...).parameterMetaData` instead of the + * `pg_proc` lookup exercised by `analyzeQuery handles CALL with parameters`. + */ + @Test + fun `analyzeQuery handles schema-qualified CALL with parameters via the prepareStatement fallback`() { + val catalog = analyzer.buildCatalog() + val parsed = ParsedQuery( + "updateStringType", + ":exec", + "CALL public.update_string_type(?, ?)", + emptyList(), + ) + + val query = analyzer.analyzeQuery(parsed, catalog) + + assertThat(query.columns).isEmpty() + assertThat(query.params).hasSize(2) + } + @Test fun `analyzeQuery preserves comments`() { val catalog = analyzer.buildCatalog() diff --git a/generator/src/test/kotlin/norm/generator/PgNodeExpressionTest.kt b/generator/src/test/kotlin/norm/generator/PgNodeExpressionTest.kt new file mode 100644 index 00000000..e3c23f4c --- /dev/null +++ b/generator/src/test/kotlin/norm/generator/PgNodeExpressionTest.kt @@ -0,0 +1,247 @@ +package norm.generator + +import assertk.assertThat +import assertk.assertions.isEmpty +import assertk.assertions.isEqualTo +import assertk.assertions.isSameInstanceAs +import org.junit.jupiter.api.Nested +import org.junit.jupiter.api.TestInstance +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.MethodSource + +/** + * One non-leaf [PgNodeExpression] subtype, constructed with a distinct sentinel + * [PgNodeExpression.Var] in every [PgNodeExpression]/[PgNodeExpression]?/`List` + * constructor parameter, so a [PgNodeExpression.children] or [PgNodeExpression.mapChildren] branch + * that mishandles a field's position or omits it entirely fails [expectedChildren]'s comparison. + */ +internal data class NonLeafCase( + val description: String, + val node: PgNodeExpression, + val expectedChildren: List, +) { + override fun toString() = description +} + +/** + * Table-driven coverage for [PgNodeExpression.children] and [PgNodeExpression.mapChildren]: one + * case per subtype, distinguishing leaves (which return an empty list / themselves unchanged) from + * every other subtype (which must enumerate its children in constructor-parameter order and rebuild + * itself around a transformed copy of them). + * + * `mapChildren { it } == node` alone would pass even when a branch forgets a field entirely — an + * identity copy of a wrong field set is still equal to the original if that field was never varied — + * so each non-leaf case also asserts [PgNodeExpression.children] directly, and that + * [PgNodeExpression.mapChildren] applies a non-identity transform to exactly those children. + */ +class PgNodeExpressionTest { + + @Nested + @TestInstance(TestInstance.Lifecycle.PER_CLASS) + internal inner class NonLeafSubtypes { + + private val replaceWithConst: (PgNodeExpression) -> PgNodeExpression = { PgNodeExpression.Const(isNull = true) } + + @ParameterizedTest(name = "{0}") + @MethodSource("nonLeafCases") + fun `children returns the exact sentinel list in constructor-parameter order`(testCase: NonLeafCase) { + assertThat(testCase.node.children).isEqualTo(testCase.expectedChildren) + } + + @ParameterizedTest(name = "{0}") + @MethodSource("nonLeafCases") + fun `mapChildren applies transform to exactly the enumerated children`(testCase: NonLeafCase) { + val mapped = testCase.node.mapChildren(replaceWithConst) + assertThat(mapped.children).isEqualTo(testCase.expectedChildren.map(replaceWithConst)) + } + + @ParameterizedTest(name = "{0}") + @MethodSource("nonLeafCases") + fun `mapChildren with the identity transform rebuilds an equal node`(testCase: NonLeafCase) { + assertThat(testCase.node.mapChildren { it }).isEqualTo(testCase.node) + } + + fun nonLeafCases(): List { + val sentinels = (1..4).map { PgNodeExpression.Var(varno = it, varattno = 0, nullingRelations = emptySet()) } + val (first, second, third, fourth) = sentinels + return listOf( + NonLeafCase( + "FuncExpr", + PgNodeExpression.FuncExpr(functionOid = 1, arguments = listOf(first), isVariadic = true), + listOf(first), + ), + NonLeafCase( + "OpExpr", + PgNodeExpression.OpExpr(operatorFunctionOid = 1, arguments = listOf(first)), + listOf(first), + ), + NonLeafCase( + "ScalarArrayOpExpr", + PgNodeExpression.ScalarArrayOpExpr(operatorFunctionOid = 1, arguments = listOf(first), useOr = true), + listOf(first), + ), + NonLeafCase( + "CoalesceExpr", + PgNodeExpression.CoalesceExpr(arguments = listOf(first)), + listOf(first), + ), + NonLeafCase( + "NullIfExpr", + PgNodeExpression.NullIfExpr(arguments = listOf(first)), + listOf(first), + ), + NonLeafCase( + "MinMaxExpr", + PgNodeExpression.MinMaxExpr(arguments = listOf(first)), + listOf(first), + ), + NonLeafCase( + "Aggref", + PgNodeExpression.Aggref(aggregateFunctionOid = 1, arguments = listOf(first)), + listOf(first), + ), + NonLeafCase( + "WindowFunc", + PgNodeExpression.WindowFunc(windowFunctionOid = 1, arguments = listOf(first)), + listOf(first), + ), + NonLeafCase( + "SubLink", + PgNodeExpression.SubLink(subLinkType = 2, outerOperand = first), + listOf(first), + ), + NonLeafCase( + "CaseExpr", + PgNodeExpression.CaseExpr( + resultExpressions = listOf(first), + defaultResult = second, + testExpression = third, + whenConditions = listOf(fourth), + ), + listOf(first, second, third, fourth), + ), + NonLeafCase( + "BoolExpr", + PgNodeExpression.BoolExpr(arguments = listOf(first), boolOperator = PgNodeExpression.BOOL_OPERATOR_AND), + listOf(first), + ), + NonLeafCase( + "RelabelType", + PgNodeExpression.RelabelType(argument = first), + listOf(first), + ), + NonLeafCase( + "CoerceViaIo", + PgNodeExpression.CoerceViaIo(argument = first), + listOf(first), + ), + NonLeafCase( + "ArrayCoerceExpr", + PgNodeExpression.ArrayCoerceExpr(argument = first), + listOf(first), + ), + NonLeafCase( + "CollateExpr", + PgNodeExpression.CollateExpr(argument = first), + listOf(first), + ), + NonLeafCase( + "CoerceToDomain", + PgNodeExpression.CoerceToDomain(argument = first), + listOf(first), + ), + NonLeafCase( + "NullTest", + PgNodeExpression.NullTest(argument = first, nullTestType = PgNodeExpression.NULL_TEST_IS_NULL), + listOf(first), + ), + NonLeafCase( + "BooleanTest", + PgNodeExpression.BooleanTest(argument = first), + listOf(first), + ), + NonLeafCase( + "DistinctExpr", + PgNodeExpression.DistinctExpr(arguments = listOf(first)), + listOf(first), + ), + NonLeafCase( + "ArrayExpr", + PgNodeExpression.ArrayExpr(elements = listOf(first)), + listOf(first), + ), + NonLeafCase( + "RowExpr", + PgNodeExpression.RowExpr(arguments = listOf(first)), + listOf(first), + ), + NonLeafCase( + "GroupingFunc", + PgNodeExpression.GroupingFunc(arguments = listOf(first)), + listOf(first), + ), + NonLeafCase( + "FieldSelect", + PgNodeExpression.FieldSelect(argument = first, fieldNumber = 1), + listOf(first), + ), + NonLeafCase( + "JsonIsPredicate", + PgNodeExpression.JsonIsPredicate(argument = first), + listOf(first), + ), + NonLeafCase( + "JsonConstructorExpr", + PgNodeExpression.JsonConstructorExpr( + type = PgNodeExpression.JSON_CONSTRUCTOR_TYPE_OBJECT, + arguments = listOf(first), + function = second, + ), + listOf(first, second), + ), + NonLeafCase( + "JsonExpr", + PgNodeExpression.JsonExpr( + op = PgNodeExpression.JSON_VALUE_OP, + argument = first, + onEmpty = PgNodeExpression.JSON_BEHAVIOR_DEFAULT, + onEmptyDefault = second, + onError = PgNodeExpression.JSON_BEHAVIOR_DEFAULT, + onErrorDefault = third, + ), + listOf(first, second, third), + ), + NonLeafCase( + "XmlExpr", + PgNodeExpression.XmlExpr(op = PgNodeExpression.XML_IS_XMLCONCAT, arguments = listOf(first)), + listOf(first), + ), + ) + } + } + + @Nested + @TestInstance(TestInstance.Lifecycle.PER_CLASS) + internal inner class LeafSubtypes { + + @ParameterizedTest(name = "{0}") + @MethodSource("leafNodes") + fun `children is empty`(node: PgNodeExpression) { + assertThat(node.children).isEmpty() + } + + @ParameterizedTest(name = "{0}") + @MethodSource("leafNodes") + fun `mapChildren with the identity transform returns the same instance`(node: PgNodeExpression) { + assertThat(node.mapChildren { it }).isSameInstanceAs(node) + } + + fun leafNodes(): List = listOf( + PgNodeExpression.Var(varno = 1, varattno = 0, nullingRelations = emptySet()), + PgNodeExpression.Const(isNull = false), + PgNodeExpression.SqlValueFunction(operation = 0), + PgNodeExpression.NextValExpr(sequenceOid = 1), + PgNodeExpression.Unknown(nodeType = "FOO"), + ) + } +} diff --git a/generator/src/test/kotlin/norm/generator/PgNodeTreeParserTest.kt b/generator/src/test/kotlin/norm/generator/PgNodeTreeParserTest.kt index f592bbd8..b45b41dd 100644 --- a/generator/src/test/kotlin/norm/generator/PgNodeTreeParserTest.kt +++ b/generator/src/test/kotlin/norm/generator/PgNodeTreeParserTest.kt @@ -181,7 +181,7 @@ class PgNodeTreeParserTest { )} )} """.trimIndent() - val result = parser.parseGroupRteExpressions(text) + val result = parser.parseRangeTableEntries(text).groupExpressions(parser) assertThat(result).hasSize(1) val groupExpressions = result.getValue(2) assertThat(groupExpressions).hasSize(1) @@ -201,7 +201,7 @@ class PgNodeTreeParserTest { {RANGETBLENTRY :rtekind 1 :relid 0} )} """.trimIndent() - assertThat(parser.parseGroupRteExpressions(text)).hasSize(0) + assertThat(parser.parseRangeTableEntries(text).groupExpressions(parser)).hasSize(0) } @Test @@ -214,7 +214,7 @@ class PgNodeTreeParserTest { {RANGETBLENTRY :eref {ALIAS :aliasname *GROUP* :colnames ("k")} :rtekind 9 :groupexprs ( {FUNCEXPR :funcid 481 :args ( """.trimIndent() - assertThat(parser.parseGroupRteExpressions(text)).hasSize(0) + assertThat(parser.parseRangeTableEntries(text).groupExpressions(parser)).hasSize(0) } @Test @@ -233,7 +233,7 @@ class PgNodeTreeParserTest { )} )} """.trimIndent() - val result = parser.parseGroupRteExpressions(text) + val result = parser.parseRangeTableEntries(text).groupExpressions(parser) val groupExpressions = result.getValue(2) assertThat(groupExpressions).hasSize(2) val first = groupExpressions[0] as PgNodeExpression.Var @@ -242,6 +242,31 @@ class PgNodeTreeParserTest { val second = groupExpressions[1] as PgNodeExpression.Const assertThat(second.isNull).isFalse() } + + @Test + fun `groupRteMap and groupExpressions intentionally disagree on a non-Var grouping expression`() { + // The grouping key is `0::bigint`'s FUNCEXPR cast, wrapping a VAR rather than being one. + // groupRteMap's textual scan reaches past the FUNCEXPR to the first :varno/:varattno it finds + // — the nested VAR's — while groupExpressions keeps the full FuncExpr node. + val text = """ + {QUERY :rtable ( + {RANGETBLENTRY :rtekind 0 :relid 24819 :relkind r} + {RANGETBLENTRY :eref {ALIAS :aliasname *GROUP* :colnames ("k")} :rtekind 9 :groupexprs ( + {FUNCEXPR :funcid 481 :funcresulttype 20 :funcretset false :funcvariadic false + :funcformat 0 :funccollid 0 :inputcollid 0 :args ( + {VAR :varno 1 :varattno 2 :vartype 25 :vartypmod -1 :varcollid 100 :varnullingrels (b) + :varlevelsup 0 :varnosyn 1 :varattnosyn 2 :location -1} + ) :location -1} + )} + )} + """.trimIndent() + val rangeTableEntries = parser.parseRangeTableEntries(text) + val groupRteMap = rangeTableEntries.groupRteMap(parser) + assertThat(groupRteMap[2 to 1]).isEqualTo(1 to 2) + val groupExpressions = rangeTableEntries.groupExpressions(parser) + val functionCall = groupExpressions.getValue(2)[0] as PgNodeExpression.FuncExpr + assertThat(functionCall.functionOid).isEqualTo(481) + } } @Nested @@ -444,7 +469,7 @@ class PgNodeTreeParserTest { :ctelevelsup 1 :self_reference false} )} """.trimIndent() - val result = parser.parseCteRangeTableEntries(text) + val result = parser.parseRangeTableEntries(text).cteReferences() assertThat(result).hasSize(1) val reference = result.getValue(1) assertThat(reference.name).isEqualTo("c") @@ -461,7 +486,7 @@ class PgNodeTreeParserTest { :self_reference false} )} """.trimIndent() - val result = parser.parseCteRangeTableEntries(text) + val result = parser.parseRangeTableEntries(text).cteReferences() assertThat(result).hasSize(1) val reference = result.getValue(1) assertThat(reference.name).isEqualTo("c") diff --git a/generator/src/test/kotlin/norm/generator/QueryAnalysisTest.kt b/generator/src/test/kotlin/norm/generator/QueryAnalysisTest.kt index f6505c9d..0cdd3466 100644 --- a/generator/src/test/kotlin/norm/generator/QueryAnalysisTest.kt +++ b/generator/src/test/kotlin/norm/generator/QueryAnalysisTest.kt @@ -24,6 +24,14 @@ import java.sql.DriverManager import java.sql.Statement import java.util.concurrent.atomic.AtomicInteger +/** + * PostgreSQL's `FirstNormalObjectId`: the first OID handed out after `initdb` finishes. Every + * `pg_proc` row below this was created with the cluster and cannot be dropped by a test, so a + * comparison restricted to this range is immune to a sibling test creating or dropping a function + * concurrently against the shared container. + */ +private const val FIRST_NORMAL_OBJECT_ID = 16384 + /** * One window function's expected result-column nullability, against the fixed one-row-per-group * shape `SELECT id, OVER(...) AS alias FROM t` -- [schema] defaults to a single @@ -760,7 +768,7 @@ class QueryAnalysisTest { * `substring(text FROM pattern)` is STRICT, yet returns `null` on a non-matching pattern even * though every input is non-null. `substring` shares its `proname` with the total * `substring(text, int, int)` overload, so the safe-list excludes the name wholesale (see - * [PgCatalogLoader.neverNullForNonNullInputOids]) rather than trying to key it by argument + * [NullabilityCatalog.neverNullForNonNullInputOids]) rather than trying to key it by argument * signature. */ @Test @@ -801,7 +809,7 @@ class QueryAnalysisTest { /** * Closes a class of unsoundness for user code: a STRICT function is no longer inferred * non-null just because it is strict. Only functions on the - * [PgCatalogLoader.neverNullForNonNullInputOids] safe-list — which is restricted to + * [NullabilityCatalog.neverNullForNonNullInputOids] safe-list — which is restricted to * `pg_catalog` — get that inference; a user-defined STRICT function in `public` does not, * because Norm cannot prove it is total on non-null input. */ @@ -1475,7 +1483,7 @@ class QueryAnalysisTest { * (`COUNT(r.id)::int AS review_count` over a `LEFT JOIN`), which that scenario's own golden * comparison test skips (it is in `NormPluginTest.EMBED_SCENARIOS`). `COUNT` is already * non-null via its non-null initial value; the `::int` cast wraps that in a FuncExpr calling - * `pg_catalog.int4(int8)`, which must be on [PgCatalogLoader.neverNullForNonNullInputOids] via + * `pg_catalog.int4(int8)`, which must be on [NullabilityCatalog.neverNullForNonNullInputOids] via * the `pg_cast` cast-function rule, or this column would regress to nullable. */ @Test @@ -1671,6 +1679,23 @@ class QueryAnalysisTest { assertThat(query.columns[0].notNull).isFalse() } + @Test + fun `plain GROUP BY key read through a derived table resolves to its base column`() { + // A plain GROUP BY (no CUBE/ROLLUP/GROUPING SETS) never null-extends its own grouping key, + // so category should resolve exactly as it would un-grouped. Two independent paths can carry + // that: substituteGroupRteVars rewrites the *GROUP* RTE Var to the grouping key's own Var + // before the fallback chain runs, and the chain's GROUP RTE remap resolves what survives + // substitution. Only the second differed between query-block resolution sites, so this pins + // that a derived table stays correct whichever one carries it. PostgreSQL 18: returns + // category NOT NULL. + val query = analyzeWithSchema( + "CREATE TABLE t (category TEXT NOT NULL)", + "SELECT s.category FROM (SELECT category FROM t GROUP BY category) s", + ) + assertThat(query.columns).hasSize(1) + assertThat(query.columns[0].notNull).isTrue() + } + // A CaseExpr's :arg (test expression) and each WHEN's :expr (condition) are ordinary // expressions that can carry the only Var in the whole CASE — but neither is evaluated by // isNonNull, since a CASE's own nullability never depends on them. Without also walking them @@ -2487,7 +2512,7 @@ class QueryAnalysisTest { @Test fun `a plain GROUP BY on a constant-folding key is non-null — no ROLLUP, still a GROUP RTE on PostgreSQL 18`() { // A GROUP RTE is created for a plain GROUP BY too, not only GROUPING SETS/CUBE/ROLLUP. - // hasGroupingSets is false here — parseGroupRteMap alone cannot resolve this key, because + // hasGroupingSets is false here — groupRteMap alone cannot resolve this key, because // its groupexprs entry is a FUNCEXPR/CONST, not a bare VAR (see that method's KDoc) — so // this exercises the substitution fix on the code path GROUPING SETS tests never touch. val query = analyzeWithSchema( @@ -2504,7 +2529,7 @@ class QueryAnalysisTest { // Before the GroupRteSubstitution fix, this was a confidently wrong NOT NULL on PostgreSQL // 18, not merely an over-widening: the target-list Var wrapping the GROUP RTE reference // carries an empty :varnullingrels (PostgreSQL does not propagate the outer join's nulling - // relations onto it), while parseGroupRteMap's coarser VAR-only resolution maps it back to + // relations onto it), while groupRteMap's coarser VAR-only resolution maps it back to // the base column by (varno, varattno) alone and discards the GROUP RTE's own :groupexprs // entry — the one that actually carries the correct, non-empty nulling relations from the // LEFT JOIN. x is NOT NULL by schema, but the join can still leave it absent for an unmatched @@ -2788,6 +2813,21 @@ class QueryAnalysisTest { assertThat(query.columns[0].notNull).isFalse() } + @Test + fun `ANY sublink whose own subselect is narrowed by its own WHERE IS NOT NULL qual is non-null`() { + // Same shape as the test above, but u's own subselect narrows itself with `WHERE v IS NOT + // NULL` — proving analyzeQueryBlockNullability's own qual gate (not merely the base + // column's own catalog constraint) still applies to a SubLink's own subselect. PostgreSQL + // 18: `a = ANY (...)` is either `true` or `false`, never `null`, since every row this + // subselect can return has a non-null v. + val query = analyzeWithSchema( + "CREATE TABLE t (id INT PRIMARY KEY, a TEXT NOT NULL); CREATE TABLE u (v TEXT)", + "SELECT a = ANY (SELECT v FROM u WHERE v IS NOT NULL) AS result FROM t", + ) + assertThat(query.columns).hasSize(1) + assertThat(query.columns[0].notNull).isTrue() + } + @Test fun `ANY sublink whose outer operand is itself a SUBLINK does not shadow the outer subselect`() { // PostgreSQL 17 and 18: `SELECT EXISTS (SELECT v FROM u) = ANY (SELECT b FROM x) FROM t` @@ -3673,8 +3713,11 @@ class QueryAnalysisTest { @Test fun `forward-referencing data-modifying CTE under WITH RECURSIVE`() { // "ins" is a data-modifying CTE whose body references "later", a CTE declared after it in - // the same WITH RECURSIVE clause. A prefix built only from preceding CTE definitions omits - // "later" and fails to prepare; the probe must use the full WITH clause instead. + // the same WITH RECURSIVE clause. The whole WITH clause — every CTE, forward references + // included — is handed to PostgreSQL verbatim inside `BEGIN ATOMIC` + // (`ColumnNullabilityAnalyzer.queryColumnNullabilityViaProsqlbody`); nothing here ever + // builds a narrower prefix of the CTE list, so PostgreSQL's own name resolution sees + // "later" regardless of declaration order. val query = analyzeWithSchema( "CREATE TABLE t (id SERIAL NOT NULL, name TEXT NOT NULL)", """ @@ -3693,10 +3736,11 @@ class QueryAnalysisTest { @Test fun `forward-referencing data-modifying CTE alongside a sibling that references it`() { - // Reproduces the mixed case that motivated using the FULL WITH clause as the probe prefix - // (rather than just extending the preceding-definitions prefix to include later CTEs): - // "ins" references the later-declared "later", while "uses" references "ins" itself. The - // full WITH clause resolves both directions at once. + // Mixed-direction companion to the test above: "ins" references the later-declared + // "later", while "uses" references "ins" itself. Both directions resolve from the same + // verbatim statement PostgreSQL parses inside `BEGIN ATOMIC` + // (`ColumnNullabilityAnalyzer.queryColumnNullabilityViaProsqlbody`) — there is no separate + // resolution pass per CTE that could see one direction and miss the other. val query = analyzeWithSchema( "CREATE TABLE t2 (id SERIAL NOT NULL, name TEXT NOT NULL)", """ @@ -3788,13 +3832,14 @@ class QueryAnalysisTest { @Test fun `data-modifying CTE body with its own nested WITH still resolves sibling shadowing correctly`() { - // "upd"'s body carries its own nested WITH ("helper") and a FROM clause, so - // convertDmlCteBodyToSelect converts it to a real join-preserving SELECT — reattaching - // "helper" verbatim in front of "SELECT src.name AS c FROM t, src, helper" — which then - // goes through the same node-tree analysis as any other CTE, resolving "src" against the - // sibling CTE (declared before "upd", so it shadows the base table normally) rather than a - // metadata probe that would be blind to this either way. Inserting a NULL row into "other" - // shows the sibling CTE "src" wins, so "c" must be nullable. + // "upd"'s body carries its own nested WITH ("helper") and its own FROM clause; the whole + // body — nested WITH included — is one node tree PostgreSQL parses inside the prosqlbody + // probe function, so "src" resolves through the same CTE-reference machinery every query + // block uses (`ColumnNullabilityAnalyzer.buildQueryBlockScope`, + // `QueryBlockScope.isSourceColumnNotNull`'s CTE branch) against the sibling CTE declared + // before "upd" (which shadows the base table of the same name), not against the base + // table's own catalog constraint. Inserting a NULL row into "other" shows the sibling CTE + // "src" wins, so "c" must be nullable. val query = analyzeWithSchema( """ CREATE TABLE other (name TEXT); @@ -3841,11 +3886,12 @@ class QueryAnalysisTest { @Test fun `no-RETURNING data-modifying CTE with its own nested WITH at a non-zero index`() { - // "logged" is the second CTE (index > 0, so the bare-body candidate that commit d4f8d7c - // relied on is never tried, by design) and has no RETURNING clause, so the true-scope - // probe ("SELECT * FROM logged") itself fails to prepare. This exercises the further - // fallback: " SELECT 1" confirms the WITH clause is otherwise sound, so a - // norm_stub is returned for "logged" instead of aborting generation. + // "logged" has no RETURNING clause and is never referenced by the outer query, so it has + // no output column for anything to resolve. `ColumnNullabilityAnalyzer.resolveCteBodies` + // resolves every CTE in the WITH clause eagerly, but silently skips one whose own + // nullability comes back empty (`analyzeCteBodyNullability`'s `?: continue`) rather than + // aborting the rest of the query's analysis — "id" and "name", which come only from "t", + // stay correctly analyzed regardless of what "logged" contains. val query = analyzeWithSchema( """ CREATE TABLE t (id SERIAL NOT NULL, name TEXT NOT NULL); @@ -3868,15 +3914,13 @@ class QueryAnalysisTest { } @Test - fun `quoted mixed-case CTE name is preserved verbatim in the true-scope probe`() { - // Proves CteDefinition.rawName (not the quote-stripped .name) is used for the - // true-scope probe: "MyIns"'s body has its own nested WITH, forcing the true-scope - // fallback ("SELECT * FROM "). If the quote-stripped name were used instead, - // "FROM MyIns" (unquoted) would fold to lowercase and fail to find the quoted, - // mixed-case relation "MyIns" — falling through to the no-RETURNING fallback and - // fabricating a single unrelated "norm_stub" column, which would then make the outer - // query's reference to "id" fail to resolve against the stubbed CTE, aborting generation - // entirely instead of merely losing nullability precision. + fun `quoted mixed-case CTE name resolves correctly when the CTE body has its own nested WITH clause`() { + // Before the prosqlbody cutover, resolving a CTE relied on splicing its name into new SQL + // text, which needed to preserve exact quoting to avoid case-folding a quoted, mixed-case + // name like "MyIns" to a different (or nonexistent) relation. The statement is handed to + // PostgreSQL whole inside `BEGIN ATOMIC` + // (`ColumnNullabilityAnalyzer.queryColumnNullabilityViaProsqlbody`); nothing re-composes SQL + // text, so quoting is preserved by construction and this shape has nothing left to break. val query = analyzeWithSchema( "CREATE TABLE t (id SERIAL NOT NULL, name TEXT NOT NULL)", """ @@ -3891,26 +3935,25 @@ class QueryAnalysisTest { """.trimIndent(), ) assertThat(query.columns).hasSize(1) - // PostgreSQL's target-list origin tracking traces "id" all the way through "SELECT * FROM - // \"MyIns\"" back to t.id, so isNullable reports NOT NULL precisely here (tableName came - // back as "t", not unknown) — a bare column RETURNING with no intervening expression - // preserves lineage. "MyIns" is a plain INSERT (no FROM/USING/MERGE join in the outer - // statement, only inside its own SELECT source), - // so it never reaches convertDmlCteBodyToSelect's join-preserving conversion and stays on - // the probe/stub path — which is fine specifically because INSERT's RETURNING sees only - // the just-inserted row: there is no outer join here for the probe to be blind to. The - // probe/stub path's fundamental blind spot to outer-join null extension (documented on - // PgCatalogLoader.buildSelectStub) does not apply to this test. + // The outer "SELECT id FROM MyIns" reads a CTE column, so isSourceColumnNotNull falls + // through to its CTE branch (ColumnNullabilityAnalyzer.kt:160-162) and takes the answer the + // recursively-analyzed CTE body already produced for "id"; within that body the RETURNING + // Var resolves through the range table to t.id's catalog attnotnull (:154). Without the + // body's own analysis "id" would run off the end of that chain and report nullable. + // "MyIns" is a plain INSERT with no outer join at all (no FROM/USING/MERGE join in the + // outer statement, only inside its own SELECT source) — RETURNING sees only the + // just-inserted row, so there is no outer-join null extension for this test to be blind to. assertThat(query.columns[0].notNull).isTrue() } @Test fun `UPDATE FROM LEFT JOIN RETURNING joined column inside a data-modifying CTE`() { - // A metadata probe (PreparedStatement.getMetaData().isNullable) reports base-table - // attnotnull — b.val is NOT NULL in the schema — and is blind to the LEFT JOIN - // null-extending it at runtime. Inserting an "a" row with no matching "b" row: the query - // returns v = NULL, so this must be nullable. Before convertDmlCteBodyToSelect existed, - // the probe/stub path reported this NOT NULL. + // b.val is declared NOT NULL, but the LEFT JOIN null-extends it at runtime for an "a" row + // with no matching "b" row. `PgNodeTreeParser.parseVar`'s `:varnullingrels` field carries + // exactly this per-column outer-join fact from PostgreSQL's own planner + // (`NodeTreeNullabilityAnalyzer`'s `isOuterJoinNullable` check), so the RETURNING Var for + // "v" is correctly reported nullable regardless of b.val's own catalog constraint. + // Inserting an "a" row with no matching "b" row: the query returns v = NULL. val query = analyzeWithSchema( """ CREATE TABLE t (id SERIAL NOT NULL, name TEXT); @@ -3930,10 +3973,10 @@ class QueryAnalysisTest { @Test fun `UPDATE FROM LEFT JOIN RETURNING joined column, body with its own nested WITH`() { - // Same shape as the test above, but "upd"'s body carries its own nested WITH ("helper"), - // exercising convertDmlCteBodyToSelect's nested-WITH reattachment path (WITH helper AS - // (...) SELECT b.val AS v FROM t, a LEFT JOIN b ON ..., helper) rather than the plain - // conversion path. Confirmed the same way: v = NULL. + // Same shape as the test above, but "upd"'s body carries its own nested WITH ("helper") of + // its own — the whole body, nested WITH included, is one node tree PostgreSQL parses and + // annotates with `:varnullingrels`, so the nested WITH changes nothing about how the LEFT + // JOIN's null extension reaches "v". Confirmed the same way: v = NULL. val query = analyzeWithSchema( """ CREATE TABLE t (id SERIAL NOT NULL, name TEXT); @@ -3954,17 +3997,14 @@ class QueryAnalysisTest { @Test fun `sibling CTE body with its own nested WITH keeps its LEFT JOIN nullability alongside a DML CTE`() { - // "j"'s body starts with its own nested WITH ("inner_cte"), so isNonDataModifyingCteBody - // must classify it by "inner_cte"'s own main statement (a plain SELECT) rather than - // stubbing "j" outright — a stub is built from PreparedStatement.getMetaData().isNullable, - // which reflects base-table attnotnull and is blind to the LEFT JOIN inside "j". Before the - // fix, "j" fell into the stub branch, whose hasOuterJoin safety net (PgCatalogLoader.kt's - // forceAllNullable) then forced every column of "j" nullable — so "did", which is - // genuinely NOT NULL, was wrongly reported nullable; "label" also came back nullable, but - // only because it was forced along with everything else, not because the stub actually saw - // the LEFT JOIN. "did" comes from the LEFT JOIN's preserved side (d), so it stays NOT NULL; - // "label" comes from the null-extended side (u), so it must be nullable. Inserting a "d" - // row with no matching "u" row: the query returns did = , label = NULL. + // "j"'s body starts with its own nested WITH ("inner_cte"), sitting alongside an unrelated + // data-modifying sibling CTE ("ins"). The whole statement, "j"'s nested WITH included, is + // one node tree resolved through `ColumnNullabilityAnalyzer.buildQueryBlockScope`, so "did" + // (from the LEFT JOIN's preserved side, d) and "label" (from the null-extended side, u) are + // each reported by their own `:varnullingrels` fact, not lumped together by the presence of + // a sibling CTE elsewhere in the query. "did" stays NOT NULL; "label" must be nullable. + // Inserting a "d" row with no matching "u" row: the query returns did = , label = + // NULL. val query = analyzeWithSchema( """ CREATE TABLE d (id INT NOT NULL); @@ -3989,15 +4029,12 @@ class QueryAnalysisTest { @Test fun `data-modifying CTE body with its own nested WITH still takes the DML path alongside a sibling CTE`() { - // Regression guard for the isNonDataModifyingCteBody classification change: a body shaped - // "WITH helper AS (...) UPDATE ... FROM ... RETURNING ..." must still be recognized as - // data-modifying (and go through convertDmlCteBodyToSelect's join-preserving conversion) - // even with an unrelated sibling CTE present — not be misclassified as verbatim-safe by - // the new nested-WITH handling. If it were misclassified, the embedded UPDATE would still - // be present when this SQL is used to CREATE VIEW, which PostgreSQL rejects, and the whole - // analysis would fall back to asserting every column NOT NULL — masking the LEFT JOIN's - // real nullability. Inserting an "a" row with no matching "b" row: the query returns v = - // NULL. + // A body shaped "WITH helper AS (...) UPDATE ... FROM ... RETURNING ..." alongside an + // unrelated sibling CTE ("seed"): the whole statement is one node tree parsed by PostgreSQL + // inside the prosqlbody probe function regardless of how many CTEs, nested or sibling, it + // contains, so "upd"'s own nested WITH and "seed"'s presence change nothing about how the + // LEFT JOIN's `:varnullingrels` reaches "v". Inserting an "a" row with no matching "b" row: + // the query returns v = NULL. val query = analyzeWithSchema( """ CREATE TABLE t (id SERIAL NOT NULL, name TEXT); @@ -4021,15 +4058,15 @@ class QueryAnalysisTest { @Test fun `parenthesized nested-WITH CTE body keeps its LEFT JOIN nullability, alongside a data-modifying CTE`() { - // Same shape as the nested-WITH CTE-body test above, but "j"'s body is additionally wrapped in its own - // parentheses (PostgreSQL accepts this: confirmed directly against a real server that - // "j AS ((WITH inner_cte AS (...) SELECT ...))" parses and returns did = 1, label = NULL - // for a "d" row with no matching "u" row). isNonDataModifyingCteBody must skip the extra - // leading "(" the same way it already does for a plain (non-nested-WITH) parenthesized - // body, then classify by the nested WITH's own main statement. A sibling data-modifying - // CTE ("ins") is required here — a query with no DML at all never reaches - // transformForViewCreation/isNonDataModifyingCteBody, since the direct CREATE VIEW - // fast path in queryColumnNullability already succeeds for it. + // Same shape as the nested-WITH CTE-body test above, but "j"'s body is additionally wrapped + // in its own extra parentheses (PostgreSQL accepts this: confirmed directly against a real + // server that "j AS ((WITH inner_cte AS (...) SELECT ...))" parses and returns did = 1, + // label = NULL for a "d" row with no matching "u" row). The extra parentheses are ordinary + // SQL syntax PostgreSQL's own parser strips while building the node tree + // `queryColumnNullabilityViaProsqlbody` reads — nothing in Norm re-parses or re-splices "j"'s + // body text — so they change nothing about how "did"/"label" resolve. A sibling + // data-modifying CTE ("ins") is included alongside "j" to confirm the extra parentheses are + // unaffected by a sibling DML statement elsewhere in the same WITH clause. val query = analyzeWithSchema( """ CREATE TABLE d (id INT NOT NULL); @@ -4075,15 +4112,14 @@ class QueryAnalysisTest { @Test fun `self-join LEFT JOIN RETURNING kills the rejected getTableName heuristic`() { - // This is the shape that rules out a metadata heuristic considered and rejected in favor - // of structural conversion: "t2" is an alias for the target table "t" itself, sitting on - // the nullable side of a LEFT JOIN. ResultSetMetaData.getTableName() reports the base - // relation "t" for t2.name — indistinguishable, by name alone, from the actual DML target - // "t" — so a heuristic keyed on "does getTableName() match the target table name" would - // conclude t2.name is not the join side and keep it fabricated NOT NULL. Structural - // conversion sidesteps this entirely: it operates on the real join structure via aliases, - // not on relation names. Inserting an "a" row with no matching "k": the query returns v = - // NULL. + // "t2" is an alias for the target table "t" itself, sitting on the nullable side of a LEFT + // JOIN. `ResultSetMetaData.getTableName()` would report the base relation "t" for t2.name — + // indistinguishable, by name alone, from the actual DML target "t" — so a heuristic keyed on + // relation names could not tell t2.name apart from the target's own (always-present) row. + // `:varnullingrels` instead identifies the nullable side by the range-table entry's own + // varno, which is distinct for "t" and its self-joined alias "t2" regardless of what table + // name each one resolves to, so t2.name is correctly reported nullable. Inserting an "a" row + // with no matching "k": the query returns v = NULL. val query = analyzeWithSchema( """ CREATE TABLE t (id SERIAL NOT NULL, k INT NOT NULL, name TEXT NOT NULL); @@ -4106,9 +4142,11 @@ class QueryAnalysisTest { @Test fun `MERGE WHEN NOT MATCHED BY SOURCE THEN DELETE RETURNING source column inside a CTE`() { assumeTrue(pgVersion.substringBefore('.').toInt() >= 17, "WHEN NOT MATCHED BY SOURCE requires PostgreSQL 17+") - // WHEN NOT MATCHED BY SOURCE fires for target rows with no matching source row — the - // shape convertMergeToSelect models as a LEFT JOIN. On real Postgres, a target row with no - // matching source row returns s.name = NULL through this RETURNING. + // WHEN NOT MATCHED BY SOURCE fires for target rows with no matching source row. + // `ColumnNullabilityAnalyzer.mergeAbsentVarnos` resolves which side of this match can be + // absent via `EXPLAIN`'s own join type (`explainMergeSideNullability`), since match- + // optionality is invisible to `:varnullingrels` on its own. On real Postgres, a target row + // with no matching source row returns s.name = NULL through this RETURNING. val query = analyzeWithSchema( """ CREATE TABLE t (id INT PRIMARY KEY, name TEXT NOT NULL); @@ -4131,9 +4169,9 @@ class QueryAnalysisTest { @Test fun `MERGE without WHEN NOT MATCHED BY SOURCE keeps source column NOT NULL`() { assumeTrue(pgVersion.substringBefore('.').toInt() >= 17, "MERGE RETURNING requires PostgreSQL 17+") - // No "WHEN NOT MATCHED BY SOURCE" clause: convertMergeToSelect models this as a plain - // (inner) join, since every row RETURNING can see has a genuine source match. On real - // Postgres, s.name is never NULL through this RETURNING. + // No "WHEN NOT MATCHED BY SOURCE" clause: every row RETURNING can see has a genuine source + // match, so `mergeAbsentVarnos` never marks the source side absent, and s.name is never + // NULL through this RETURNING. val query = analyzeWithSchema( """ CREATE TABLE t (id INT PRIMARY KEY, name TEXT NOT NULL); @@ -4158,9 +4196,10 @@ class QueryAnalysisTest { // Checked via psql \gdesc on the actual UPDATE: "RETURNING *" on UPDATE ... FROM is not // limited to the target table's columns — it expands to every relation in the statement's // scope, target and joined, identically to a plain "SELECT *" over the same FROM list - // (t.id, t.name, a.id, a.label — 4 columns, not 2). This is why convertDmlToSelect passes - // RETURNING clauses through verbatim rather than qualifying a bare "*" to the target alone - // (which would have produced the wrong column count here). + // (t.id, t.name, a.id, a.label — 4 columns, not 2). PostgreSQL itself expands the star + // while building the RETURNING list this analysis reads + // (`PgNodeTreeParser.parseReturningList`); nothing here re-derives the star's expansion from + // the target table alone. val query = analyzeWithSchema( """ CREATE TABLE t (id INT NOT NULL, name TEXT NOT NULL); @@ -4207,10 +4246,11 @@ class QueryAnalysisTest { @Test fun `LEFT JOIN RETURNING joined column survives an unbalanced parenthesis inside a string literal`() { - // Regression guard: findTopLevelKeyword previously counted parens inside string literals, - // so the "(" inside '\(' hid the real FROM from convertDmlToSelect, silently falling back - // to the metadata probe/stub path — which is blind to the LEFT JOIN and fabricates NOT - // NULL. Inserting an "a" row with no matching "b" row: the query returns v = NULL. + // History: a text-based scan for the statement's own FROM clause once mistook the "(" inside + // '\(' for a real parenthesis. The statement is handed to PostgreSQL whole inside + // `BEGIN ATOMIC` (`ColumnNullabilityAnalyzer.queryColumnNullabilityViaProsqlbody`); nothing + // re-composes SQL text, so a parenthesis inside a string literal cannot mislead anything + // here. Inserting an "a" row with no matching "b" row: the query returns v = NULL. val query = analyzeWithSchema( """ CREATE TABLE t (id SERIAL NOT NULL, name TEXT); @@ -4233,9 +4273,11 @@ class QueryAnalysisTest { @Test fun `MERGE RETURNING source column survives an unbalanced parenthesis inside a string literal`() { assumeTrue(pgVersion.substringBefore('.').toInt() >= 17, "WHEN NOT MATCHED BY SOURCE requires PostgreSQL 17+") - // Same defect as above, for MERGE: the "(" inside a SET expression's string literal must - // not hide the real WHEN NOT MATCHED BY SOURCE clause from convertMergeToSelect. On real - // Postgres, a target row with no matching source row returns sname = NULL. + // Same historical shape as above, for MERGE: a "(" inside a SET expression's string literal + // cannot mislead anything today, since the whole statement is handed to PostgreSQL verbatim + // and match-optionality is resolved by `mergeAbsentVarnos`'s own `EXPLAIN` call, never by a + // text scan of the statement. On real Postgres, a target row with no matching source row + // returns sname = NULL. val query = analyzeWithSchema( """ CREATE TABLE mt (tid INT PRIMARY KEY, tname TEXT NOT NULL); @@ -4258,11 +4300,12 @@ class QueryAnalysisTest { @Test fun `a string literal containing the word FROM with no real FROM clause does not abort generation`() { - // Regression guard: before the lexer fix, an unvalidated conversion could replace - // PostgreSQL's own parse with garbled text derived from misreading "from" inside a string - // literal as if it introduced a real FROM clause — aborting generation entirely on SQL - // PostgreSQL accepts fine. On real Postgres, id = 1 (NOT NULL, as expected for a SERIAL - // primary key) — there is no join here at all, real or otherwise. + // History: a text-based rewrite once misread "from" inside a string literal as if it + // introduced a real FROM clause, corrupting the statement it built for analysis. The + // statement is handed to PostgreSQL whole inside `BEGIN ATOMIC` + // (`ColumnNullabilityAnalyzer.queryColumnNullabilityViaProsqlbody`); nothing re-composes SQL + // text, so this shape has nothing left to misread. On real Postgres, id = 1 (NOT NULL, as + // expected for a SERIAL primary key) — there is no join here at all, real or otherwise. val query = analyzeWithSchema( "CREATE TABLE t (id SERIAL NOT NULL, name TEXT NOT NULL)", """ @@ -4278,12 +4321,12 @@ class QueryAnalysisTest { @Test fun `a line comment containing FROM between SET and the real FROM clause does not abort generation`() { - // The comment must sit between "SET ..." and the real "FROM" — a comment before "UPDATE" - // is already skipped by the leading-whitespace/comment handling every DML-recognition - // check starts with, on both old and new code, so it would not exercise this bug (that - // shape doesn't demonstrate anything). This one forces findTopLevelKeyword to scan through - // the comment while searching for the real FROM. Inserting an "a" row with no matching "b" - // row: the query returns v = NULL. + // History: a text-based scan for the real FROM clause once needed to skip over a line + // comment sitting between "SET ..." and "FROM" to avoid stopping at the word "FROM" inside + // the comment. The statement is handed to PostgreSQL whole inside `BEGIN ATOMIC` + // (`ColumnNullabilityAnalyzer.queryColumnNullabilityViaProsqlbody`); nothing re-composes SQL + // text, so a comment's contents cannot mislead anything here. Inserting an "a" row with no + // matching "b" row: the query returns v = NULL. val query = analyzeWithSchema( """ CREATE TABLE t (id SERIAL NOT NULL, name TEXT); @@ -4304,8 +4347,8 @@ class QueryAnalysisTest { @Test fun `a block comment containing FROM between SET and the real FROM clause does not abort generation`() { - // Same reasoning as the line-comment variant above. Inserting an "a" row with no matching - // "b" row: the query returns v = NULL. + // Same historical reasoning as the line-comment variant above. Inserting an "a" row with no + // matching "b" row: the query returns v = NULL. val query = analyzeWithSchema( """ CREATE TABLE t (id SERIAL NOT NULL, name TEXT); @@ -4352,9 +4395,10 @@ class QueryAnalysisTest { // Checked via \gdesc plus executing the query: "RETURNING *" expands source-first — sid, // sname, tid, tname — regardless of which WHEN clauses are present, and for a target row // with no matching source row the actual returned values are [NULL, NULL, 1, 'target-row']. - // convertMergeToSelect must emit "FROM source RIGHT JOIN target" (source first) to match — - // a target-first conversion would report the nullability for the wrong columns even though - // the metadata (names/types) could look plausible. + // `PgNodeTreeParser.parseReturningList` reads PostgreSQL's own already-expanded star in this + // exact order, and each entry's own `:varnullingrels`/`mergeAbsentVarnos` answer decides its + // nullability independently of its position, so the source-first order is preserved without + // Norm ever re-deriving which relation comes first. val query = analyzeWithSchema( """ CREATE TABLE mt (tid INT PRIMARY KEY, tname TEXT NOT NULL); @@ -4381,9 +4425,12 @@ class QueryAnalysisTest { @Test fun `literal text matching the WHEN NOT MATCHED BY SOURCE phrase does not trigger the LEFT JOIN model`() { assumeTrue(pgVersion.substringBefore('.').toInt() >= 17, "MERGE RETURNING requires PostgreSQL 17+") - // hasWhenNotMatchedBySourceClause must not misfire on a SET expression's string literal - // that happens to contain the phrase "when not matched by source ". On real Postgres, with - // no genuine WHEN NOT MATCHED BY SOURCE clause, ms.sname is never NULL. + // History: a text-based scan for the WHEN NOT MATCHED BY SOURCE clause once could misfire + // on a SET expression's string literal that happens to contain the same phrase. The + // statement is handed to PostgreSQL whole inside `BEGIN ATOMIC` + // (`ColumnNullabilityAnalyzer.queryColumnNullabilityViaProsqlbody`); nothing re-composes SQL + // text, so a literal's contents cannot mislead anything here. On real Postgres, with no + // genuine WHEN NOT MATCHED BY SOURCE clause, ms.sname is never NULL. val query = analyzeWithSchema( """ CREATE TABLE mt (tid INT PRIMARY KEY, tname TEXT NOT NULL); @@ -4405,16 +4452,15 @@ class QueryAnalysisTest { @Test fun `data-modifying CTE preceded by a sibling CTE containing a closing parenthesis in a literal`() { - // End-to-end companion to the SqlUtilsTest paren-in-literal coverage: the first CTE's body - // contains a ')' inside a string literal, which (before the lexer fix) corrupted - // findMatchingCloseParenthesis's body-boundary detection for that CTE — parseCteClause - // then stopped after that one (corrupted) definition, treating "upd" as part of the - // garbled main-query text instead of a second CTE. "upd" has a LEFT JOIN specifically so - // this is visible: the garbled-query fallback (the top-level no-join-structure DML path, - // "assume every column non-null" before the not-null-fallback fix) happens to give the - // right answer for a plain INSERT (as in the SqlUtilsTest e2e companion above), but gives - // the wrong answer here, where the true answer is nullable. Inserting an "a" row with no - // matching "b" row: the query returns v = NULL. + // History: correctly finding a CTE body's own closing parenthesis, even with a ')' inside a + // string literal, once mattered for the nullability answer itself, when a text-based route + // built a stand-in SELECT from that boundary. `SqlCteClause.parseCteClause` and + // `findMatchingCloseParenthesis` still parse CTE boundaries today, but only to resolve + // provenance text (`NodeTreeProvenanceExpression`), never to build anything the nullability + // answer is computed from — the statement is handed to PostgreSQL whole inside + // `BEGIN ATOMIC` (`ColumnNullabilityAnalyzer.queryColumnNullabilityViaProsqlbody`) regardless + // of what any literal contains. Inserting an "a" row with no matching "b" row: the query + // returns v = NULL. val query = analyzeWithSchema( """ CREATE TABLE t (id SERIAL NOT NULL, name TEXT); @@ -4437,9 +4483,9 @@ class QueryAnalysisTest { @Test fun `sibling CTE with closing paren in a literal still generates correctly for a plain INSERT`() { - // Companion to the LEFT JOIN variant above and to the SqlUtilsTest unit coverage: proves - // the fix for a shape with NO join at all, where the pre-fix bug's corruption happened to - // be masked by the "assume non-null" fallback rather than causing a visibly wrong answer. + // Companion to the LEFT JOIN variant above, with no join at all: this shape has no + // `:varnullingrels` to be blind to in the first place, so it is exercised here purely for + // completeness alongside the CTE-boundary paren-in-literal case above. val query = analyzeWithSchema( "CREATE TABLE t (id SERIAL NOT NULL, name TEXT NOT NULL)", """ @@ -4458,12 +4504,12 @@ class QueryAnalysisTest { @Test fun `LEFT JOIN RETURNING joined column survives a SET-clause column named valid_from`() { - // Regression guard: the "_" in "valid_from" did not count as an identifier character in - // findTopLevelKeyword's word-boundary check, so "valid_from" matched the keyword "FROM" - // at its own position — before the real "FROM a LEFT JOIN b" clause — corrupting - // conversion (which then failed validation) and falling back to the metadata probe/stub, - // which is blind to the LEFT JOIN and fabricated NOT NULL. Inserting an "a" row with no - // matching "b" row: the query returns bval = NULL. + // History: a text-based scan for the real FROM clause once treated "_" as ending an + // identifier, so "valid_from" matched the keyword "FROM" at its own position, before the + // genuine "FROM a LEFT JOIN b" clause. The statement is handed to PostgreSQL whole inside + // `BEGIN ATOMIC` (`ColumnNullabilityAnalyzer.queryColumnNullabilityViaProsqlbody`); nothing + // re-composes SQL text, so a column named "valid_from" cannot mislead anything here. + // Inserting an "a" row with no matching "b" row: the query returns bval = NULL. val query = analyzeWithSchema( """ CREATE TABLE t (id INT NOT NULL, valid_from TEXT); @@ -4485,13 +4531,13 @@ class QueryAnalysisTest { @Test fun `SET-clause column named returning_note no longer aborts generation`() { - // Regression guard: before the word-boundary fix, "returning_note" matched "RETURNING" as - // a keyword, making returningIndex point inside the SET clause — earlier than the join - // clause start computed from the (correctly found) later FROM — and - // buildSelectFromDml's substring(joinClauseStart, returningIndex) threw - // StringIndexOutOfBoundsException, aborting generation on SQL PostgreSQL itself accepts - // fine. On real Postgres, id = 1 (NOT NULL, as expected for a plain UPDATE with no outer - // join at all). + // History: a text-based scan once matched "returning_note" as the keyword "RETURNING", + // corrupting the boundaries it computed for a stand-in SELECT and throwing a + // `StringIndexOutOfBoundsException` — aborting generation on SQL PostgreSQL itself accepts + // fine. The statement is handed to PostgreSQL whole inside `BEGIN ATOMIC` + // (`ColumnNullabilityAnalyzer.queryColumnNullabilityViaProsqlbody`); nothing re-composes SQL + // text, so a column named "returning_note" cannot mislead anything here. On real Postgres, + // id = 1 (NOT NULL, as expected for a plain UPDATE with no outer join at all). val query = analyzeWithSchema( """ CREATE TABLE t (id INT NOT NULL, returning_note TEXT); @@ -4509,21 +4555,19 @@ class QueryAnalysisTest { } @Test - fun `stub path forces every column nullable when RETURNING OLD-col accompanies a real LEFT JOIN`() { + fun `RETURNING OLD-col is nullable by rule while a separate LEFT JOIN column is nullable by its own real join`() { assumeTrue(pgVersion.substringBefore('.').toInt() >= 18, "RETURNING OLD requires PostgreSQL 18+") - // PostgreSQL 18's RETURNING OLD.col forces this body onto the stub path: the structural - // conversion builds a plain SELECT where "OLD" is not a valid range variable, so it fails - // to prepare and validatedConversion correctly rejects it. Before the stub-path safety - // net, the metadata probe reported the unrelated sibling column b.bval as NOT NULL (blind - // to the real LEFT JOIN elsewhere in the same body) even though oldname's own OLD-based - // imprecision was already an accepted limitation. Inserting an "a" row with no matching - // "b" row: oldname = 'orig' (the target row always exists for a plain UPDATE, so OLD.name - // is never actually null here), bval = NULL. The safety net deliberately - // over-approximates — marking every stub column nullable once any outer join is detected - // in the body, not just the ones actually reached through it — so oldname is also reported - // nullable here even though its true answer is NOT NULL: safe-direction imprecision, not a - // regression, and a documented tradeoff (see PgCatalogLoader's buildSelectStub and - // tryPrepareStub KDoc). + // PostgreSQL 18's RETURNING OLD.col is read as a `Var` whose `:varreturningtype` tags it OLD + // (`PgNodeExpression.Var.returningType`); `NodeTreeNullabilityAnalyzer.isNonNull` treats + // every such reference as nullable unconditionally, regardless of whether the OLD row + // genuinely always exists for this statement (a plain UPDATE's target row always exists, so + // OLD.name is never actually null here — this rule is deliberately blanket, not statement- + // kind-aware; see `PgNodeExpression.Var.returningType`'s own KDoc). "bval" is nullable for an + // entirely separate, precise reason: it comes through the real LEFT JOIN, and PostgreSQL's + // own `:varnullingrels` on that `Var` marks it null-extended. The two columns land on the + // same nullable answer through two independent mechanisms, not one shared + // over-approximation. Inserting an "a" row with no matching "b" row: oldname = 'orig', bval + // = NULL. val query = analyzeWithSchema( """ CREATE TABLE t (id INT NOT NULL, name TEXT NOT NULL); @@ -4546,11 +4590,13 @@ class QueryAnalysisTest { @Test fun `MERGE detects WHEN NOT MATCHED BY SOURCE despite a comment abutting NOT and MATCHED`() { assumeTrue(pgVersion.substringBefore('.').toInt() >= 17, "WHEN NOT MATCHED BY SOURCE requires PostgreSQL 17+") - // skipOptionalKeyword previously required literal whitespace immediately after each - // keyword, so a comment directly abutting NOT and MATCHED with no surrounding whitespace - // broke clause detection entirely, choosing a plain JOIN and fabricating NOT NULL for the - // source column. On real Postgres, id = 1 (NOT NULL, target row), sval = NULL (nullable, - // no matching source row). + // History: a text-based scan for WHEN NOT MATCHED BY SOURCE once required literal + // whitespace immediately around each keyword, so a comment directly abutting NOT and + // MATCHED broke detection. The statement is handed to PostgreSQL whole inside + // `BEGIN ATOMIC` (`ColumnNullabilityAnalyzer.queryColumnNullabilityViaProsqlbody`), and + // `mergeAbsentVarnos` resolves match-optionality via `EXPLAIN`'s own join type, never a text + // scan of the clause — so a comment between keywords cannot mislead anything here. On real + // Postgres, id = 1 (NOT NULL, target row), sval = NULL (nullable, no matching source row). val query = analyzeWithSchema( """ CREATE TABLE tgt (id INT PRIMARY KEY, name TEXT NOT NULL); @@ -4573,13 +4619,14 @@ class QueryAnalysisTest { @Test fun `INSERT with a LEFT JOIN in its own SELECT source reports NOT NULL, not fabricated nullable`() { - // Regression guard: the stub-path safety net previously fired for any body with a - // detectable outer join, INSERT included — but an INSERT's RETURNING sees only the row - // just inserted, and nothing in its own SELECT source (however joined) can null-extend - // it. On real Postgres, INSERT INTO b(id, bval) SELECT a.id, 'v' FROM a LEFT JOIN b2 ON - // b2.id = a.id RETURNING id, bval returns id=1, bval='v' — both non-null — despite the - // LEFT JOIN in its source. See the companion test below for the UPDATE shape, where the - // net must still fire. + // "id" and "bval" are RETURNING references to the just-inserted row's own assigned values + // (`ColumnNullabilityAnalyzer.analyzeNodeTree`'s `targetListByResno` substitution), not to + // the SELECT source's own output columns — a LEFT JOIN inside that source's `FROM` clause + // produces `:varnullingrels` scoped to the source subquery, which never propagates to the + // INSERT's own target-list assignment. On real Postgres, INSERT INTO b(id, bval) SELECT + // a.id, 'v' FROM a LEFT JOIN b2 ON b2.id = a.id RETURNING id, bval returns id=1, bval='v' — + // both non-null — despite the LEFT JOIN in its source. See the companion test below for + // the UPDATE shape, where the LEFT JOIN's null extension genuinely does reach RETURNING. val query = analyzeWithSchema( """ CREATE TABLE a (id INT NOT NULL); @@ -4601,9 +4648,10 @@ class QueryAnalysisTest { @Test fun `UPDATE with a LEFT JOIN still reports nullable — the safety net must keep working`() { - // Companion to the INSERT test above: confirms excluding INSERT from the safety net did - // not also (over-broadly) exclude UPDATE, which genuinely needs it. Inserting an "a" row - // with no matching "b" row: the query returns bval = NULL. + // Companion to the INSERT test above: confirms the same LEFT JOIN, reached through an + // UPDATE's own `FROM` clause rather than an INSERT's `SELECT` source, produces + // `:varnullingrels` on `b.bval`'s own `Var` in RETURNING, unlike the INSERT case above. + // Inserting an "a" row with no matching "b" row: the query returns bval = NULL. val query = analyzeWithSchema( """ CREATE TABLE t (id INT NOT NULL, name TEXT); @@ -4625,11 +4673,12 @@ class QueryAnalysisTest { @Test fun `LEFT JOIN RETURNING joined column survives a SET-clause column named with two dollar signs`() { - // Regression guard: the "$" between "b" and "c" in "a$b$c" was misread as opening a - // "$b$"-tagged dollar-quote, swallowing the rest of the statement — including the real - // "FROM a LEFT JOIN b" — as unterminated string content. Conversion then failed (or - // produced garbage), falling back to the metadata probe/stub, which is blind to the LEFT - // JOIN. Inserting an "a" row with no matching "b" row: the query returns bval = NULL. + // History: a text-based scan once misread the "$" between "b" and "c" in "a$b$c" as + // opening a "$b$"-tagged dollar-quote, swallowing the rest of the statement as unterminated + // string content. The statement is handed to PostgreSQL whole inside `BEGIN ATOMIC` + // (`ColumnNullabilityAnalyzer.queryColumnNullabilityViaProsqlbody`); nothing re-composes SQL + // text, so a column named "a$b$c" cannot mislead anything here. Inserting an "a" row with + // no matching "b" row: the query returns bval = NULL. val query = analyzeWithSchema( """ CREATE TABLE t (id INT NOT NULL, a${'$'}b${'$'}c TEXT); @@ -4652,15 +4701,13 @@ class QueryAnalysisTest { @Test fun `MERGE with a LEFT JOIN nested in its USING subquery reports the joined column nullable`() { assumeTrue(pgVersion.substringBefore('.').toInt() >= 17, "merge_action() requires PostgreSQL 17+") - // Regression guard: hasOuterJoin previously scanned only paren depth 0, so a LEFT JOIN - // nested inside the USING subquery went undetected — merge_action() in RETURNING already - // forces this body onto the stub path (it isn't valid outside MERGE's own RETURNING, so - // conversion to a plain SELECT fails to prepare and is rejected), and the stub then - // fabricated NOT NULL for the joined column. With sx having no row matching src: act = - // 'UPDATE', id = 1, xval = NULL. The safety net's over-approximation - // also demotes "id" (the target's PK, always present for a MATCHED row) to nullable here — - // an accepted, documented tradeoff, since the stub cannot isolate which columns are - // actually reached through the nested join (see buildSelectStub's KDoc). + // The MERGE's USING source is a subquery, not a base table or CTE, so + // `ColumnNullabilityAnalyzer.mergeSourceRelationNameCandidates` cannot name it and returns + // `null`; `mergeAbsentVarnos` propagates that, and the CTE body's own analysis returns + // `null` in turn. `resolveCteBodies` skips a body it could not resolve, so the outer query's + // lookup for "m" misses and all three columns fall back to nullable — "xval" among them. + // The reported answer is therefore the safe direction rather than a proof about this join. + // With sx having no row matching src: act = 'UPDATE', id = 1, xval = NULL. val query = analyzeWithSchema( """ CREATE TABLE tgt (id INT PRIMARY KEY, tval TEXT NOT NULL); @@ -4684,15 +4731,14 @@ class QueryAnalysisTest { @Test fun `DELETE RETURNING OLD-col alongside an unrelated column no longer drags it into nullable`() { assumeTrue(pgVersion.substringBefore('.').toInt() >= 18, "RETURNING OLD requires PostgreSQL 18+") - // Regression guard: at e4679ff, forceAllNullable applied to the whole stub once any - // RETURNING item referenced OLD/NEW — so "id" (never touched by OLD/NEW at all) was - // fabricated nullable purely because it shared a RETURNING list with "oldname". On real - // Postgres, DELETE FROM t WHERE id = 1 RETURNING OLD.name, t.id returns oldname = 'orig' - // and id = 1 — both genuinely NOT NULL for this exact row, but "id" is the one this fix - // must stop fabricating nullable for; "oldname" itself is still forced nullable - // (over-approximating in the safe direction, unchanged) since knowing OLD is genuinely - // never-null for a DELETE specifically would require statement-kind-aware logic this fix - // does not add — see oldOrNewReturningColumns's KDoc. + // "id" is never touched by OLD/NEW at all: `NodeTreeNullabilityAnalyzer.isNonNull`'s + // OLD/NEW forcing is per-`Var` (keyed on that `Var`'s own `:varreturningtype`), not a + // whole-statement rule, so sharing a RETURNING list with "oldname" cannot drag "id" into + // nullable. On real Postgres, DELETE FROM t WHERE id = 1 RETURNING OLD.name, t.id returns + // oldname = 'orig' and id = 1 — both genuinely NOT NULL for this exact row, but "oldname" + // is still reported nullable by the blanket OLD-forcing rule (see + // `PgNodeExpression.Var.returningType`'s KDoc for why it is deliberately statement-kind- + // agnostic). val query = analyzeWithSchema( "CREATE TABLE t (id INT NOT NULL, name TEXT NOT NULL)", """ @@ -4808,7 +4854,7 @@ class QueryAnalysisTest { // PostgreSQL 18's RETURNING WITH (OLD AS o, NEW AS n) prologue declares custom names for the // pseudo-relations; PgNodeTreeParser.parseVar reads :varreturningtype directly off the // Var node regardless of which alias the SQL text used, so "o"/"n" need no special - // recognition of their own the way the old text-based oldOrNewReturningColumns needed. + // recognition of their own the way an older, text-based mechanism once needed. // prosqlbody's NEW-tagged Var for "n.name" is a plain, ordinary reference for an UPDATE // (forceNewNullable only applies to DELETE and a MERGE with a DELETE action — see // NodeTreeNullabilityAnalyzer's own KDoc) — since "name" is declared NOT NULL, it correctly @@ -4836,15 +4882,13 @@ class QueryAnalysisTest { @Test fun `MERGE fed by a sibling CTE with an internal LEFT JOIN forces the joined column nullable`() { assumeTrue(pgVersion.substringBefore('.').toInt() >= 17, "merge_action() requires PostgreSQL 17+") - // "pre" (a sibling CTE, not part of "m"'s own body text) contains a LEFT JOIN whose - // null-extension is entirely invisible to any scan of "m"'s own text — "m" itself has no - // join at all. merge_action() in RETURNING forces this body onto the stub path (not valid - // outside MERGE's own RETURNING, so the structural conversion fails to prepare and is - // rejected); before this fix, the stub's base-table attnotnull fabricated "bval" as NOT - // NULL despite the real LEFT JOIN living in "pre". With an "a" row with no matching "b" - // row: act = 'UPDATE', bval = NULL, id = 1 (the target's own PK, always present for a - // MATCHED row — also demoted to nullable here, an accepted tradeoff, same as the existing - // nested-USING-subquery LEFT JOIN test above). + // "pre" (a sibling CTE) contains a LEFT JOIN; "m" itself has no join of its own and merely + // reads "pre.bval" through the CTE. `ColumnNullabilityAnalyzer.resolveCteBodies` resolves + // "pre" first and records its per-column nullability, so when "m"'s own analysis reaches a + // `Var` referencing "pre.bval" it takes that already-resolved answer + // (`QueryBlockScope.isSourceColumnNotNull`'s CTE branch) rather than anything derived from + // "m"'s own text. With an "a" row with no matching "b" row: act = 'UPDATE', bval = NULL, id + // = 1. val query = analyzeWithSchema( """ CREATE TABLE tgt (id INT PRIMARY KEY, tval TEXT NOT NULL); @@ -4870,11 +4914,12 @@ class QueryAnalysisTest { @Test fun `MERGE fed by a double-quoted sibling reference still forces the joined column nullable`() { assumeTrue(pgVersion.substringBefore('.').toInt() >= 17, "merge_action() requires PostgreSQL 17+") - // Quoting an otherwise unremarkable lowercase sibling name ("pre") used to defeat - // referencesAnyName entirely, since its underlying scan skipped double-quoted identifiers - // as an opaque lexical token by design. referencesAnyName now scans a quoted identifier's - // own contents instead. PostgreSQL 18, with an "a" row with no matching "b" row and target - // "tgt" row id = 1: act = 'UPDATE', bval = NULL, id = 1. + // Quoting an otherwise unremarkable lowercase sibling name ("pre") changes nothing about + // how "m" resolves it: PostgreSQL's own parser folds the quoted reference to the same CTE + // range-table entry regardless of quoting, and + // `ColumnNullabilityAnalyzer.resolveCteBodies` already resolved "pre"'s own nullability + // before "m" is ever analyzed. PostgreSQL 18, with an "a" row with no matching "b" row and + // target "tgt" row id = 1: act = 'UPDATE', bval = NULL, id = 1. val query = analyzeWithSchema( """ CREATE TABLE tgt (id INT PRIMARY KEY, tval TEXT NOT NULL); @@ -4902,9 +4947,12 @@ class QueryAnalysisTest { assumeTrue(pgVersion.substringBefore('.').toInt() >= 17, "merge_action() requires PostgreSQL 17+") // A ROLLUP supertotal row makes the grouped column NULL by definition, matched into the // target only via a COALESCE in the ON condition -- no LEFT/RIGHT/FULL JOIN keyword and no - // WHEN NOT MATCHED BY SOURCE clause appears anywhere in the body for the pre-existing - // detectors to find. hasGroupingSetConstruct now recognizes ROLLUP/CUBE/GROUPING SETS as a - // third null-extending construct. PostgreSQL 18, with an "a" row with id = 1 and tgt rows + // WHEN NOT MATCHED BY SOURCE clause appears anywhere in the body. + // The USING source is a subquery, so `ColumnNullabilityAnalyzer.mergeSourceRelationNameCandidates` + // cannot name it and returns `null`; the CTE body's analysis returns `null` in turn and + // `resolveCteBodies` skips it, leaving every column nullable. "sid" is reported nullable by + // that fallback, not by any ROLLUP-specific reasoning. PostgreSQL 18, with an "a" + // row with id = 1 and tgt rows // id = 1 and id = 2: the id = 1 row of the source matches tgt id = 1 (sid = 1, not the // supertotal), and the ROLLUP supertotal row (s.id = NULL) matches tgt id = 2 via // COALESCE(s.id, 2) = 2 -- two result rows, merge_action = 'UPDATE' for both, {sid = 1, id @@ -4933,11 +4981,13 @@ class QueryAnalysisTest { fun `MERGE fed by a transitive sibling chain forces the joined column nullable`() { assumeTrue(pgVersion.substringBefore('.').toInt() >= 17, "merge_action() requires PostgreSQL 17+") // "m" references "mid", which has no join of its own -- the LEFT JOIN lives in "j", which - // is "mid"'s sibling, not "m"'s. The pre-existing one-level-deep sibling-danger check - // stopped at "mid" and never saw "j". computeDangerousSiblingNames now computes the danger - // set as a fixpoint over the whole WITH clause, so "mid" (which references "j") joins the - // dangerous set first, then "m" (which references "mid") joins next. PostgreSQL 18, with - // an "a" row with no matching "b" row: act = 'UPDATE', bval = NULL, id = 1. + // is "mid"'s sibling, not "m"'s. `ColumnNullabilityAnalyzer.resolveCteBodies` resolves each + // CTE in declaration order, feeding each one's already-resolved nullability forward as + // `previouslyResolved` -- "j" resolves first (bval nullable via its own LEFT JOIN), then + // "mid" (a plain passthrough of "j", so still nullable), then "m" (a plain passthrough of + // "mid") -- so the chain propagates regardless of how many sibling CTEs sit between the + // join and the reference. PostgreSQL 18, with an "a" row with no matching "b" row: act = + // 'UPDATE', bval = NULL, id = 1. val query = analyzeWithSchema( """ CREATE TABLE tgt (id INT PRIMARY KEY, tval TEXT NOT NULL); @@ -4989,7 +5039,7 @@ class QueryAnalysisTest { @Test fun `MERGE fed by a CTE source correctly reports the passed-through column NOT NULL`() { assumeTrue(pgVersion.substringBefore('.').toInt() >= 17, "merge_action() requires PostgreSQL 17+") - // PgCatalogLoader.mergeAbsentVarnos now attributes a MERGE's join to a CTE source too + // ColumnNullabilityAnalyzer.mergeAbsentVarnos now attributes a MERGE's join to a CTE source too // (previously only a plain base table), via the CTE's own literal name -- "ins" appears // directly as a "CTE Scan" node's "CTE Name" here, since a data-modifying CTE is never // inlined. With an "a" row and no matching "b" row, the INSERT inserts one row into @@ -5023,18 +5073,16 @@ class QueryAnalysisTest { @Test fun `MERGE fed by an INSERT ON CONFLICT sibling that RETURNS OLD-col forces the passed-through column nullable`() { assumeTrue(pgVersion.substringBefore('.').toInt() >= 18, "RETURNING OLD requires PostgreSQL 18+") - // A follow-up finding: the seed's "!isInsertBody" exclusion (added for the precision - // guard test above) is correct for an INSERT's ordinary target-column RETURNING, but an - // INSERT's RETURNING can also read OLD./NEW., whose own conditional existence attnotnull - // cannot see regardless of statement kind -- excluding every INSERT body from the seed, - // rather than only excluding hasNullExtendingConstruct's own-join trigger, silently dropped - // this danger sign. computeDangerousSiblingNames now seeds separately on - // oldOrNewReturningColumns (which also understands a RETURNING WITH (OLD AS alias, ...) - // prologue), regardless of isInsertBody. "ins" is an INSERT ... ON CONFLICT DO UPDATE - // RETURNING OLD.val -- OLD is NULL exactly when the row was freshly inserted (no prior - // conflict) -- and "m" merely passes ins.oldval through. PostgreSQL 18, with an "a" row - // with no matching "b" row and no pre-existing "it2" row so the INSERT always takes the - // fresh-insert branch: act = 'UPDATE', ov = NULL, id = 1. + // "ins" is an INSERT ... ON CONFLICT DO UPDATE RETURNING id, OLD.val AS oldval; "m" merely + // passes ins.oldval through a MERGE. `ColumnNullabilityAnalyzer.resolveCteBodies` resolves + // "ins" first: its RETURNING `Var` for OLD.val is tagged by `:varreturningtype` + // (`PgNodeExpression.Var.returningType`), so the blanket OLD-forcing rule + // (`NodeTreeNullabilityAnalyzer.isNonNull`) reports "oldval" nullable regardless of "ins" + // being an INSERT rather than an UPDATE/DELETE/MERGE -- "m" then inherits that already- + // resolved nullability for "ov" through the ordinary CTE-reference chain, the same as any + // other passed-through CTE column. PostgreSQL 18, with an "a" row with no matching "b" row + // and no pre-existing "it2" row so the INSERT always takes the fresh-insert branch: act = + // 'UPDATE', ov = NULL, id = 1. val query = analyzeWithSchema( """ CREATE TABLE tgt (id INT PRIMARY KEY, tval TEXT NOT NULL); @@ -5063,11 +5111,11 @@ class QueryAnalysisTest { @Test fun `MERGE fed by an INSERT sibling using the RETURNING WITH OLD-alias prologue forces the column nullable`() { assumeTrue(pgVersion.substringBefore('.').toInt() >= 18, "RETURNING OLD requires PostgreSQL 18+") - // Same danger sign as the unqualified-OLD sibling test above, but via PostgreSQL 18's + // Same shape as the unqualified-OLD sibling test above, but via PostgreSQL 18's // `RETURNING WITH (OLD AS alias, ...)` prologue instead of a bare `OLD.col` reference -- - // computeDangerousSiblingNames seeds on oldOrNewReturningColumns specifically because it - // (unlike a bare referencesOldOrNew call) already understands this prologue, so an aliased - // reference must trip the same seed. "ins" is an INSERT ... ON CONFLICT DO UPDATE + // `PgNodeTreeParser.parseVar` reads `:varreturningtype` directly off the `Var` node + // regardless of which alias the SQL text declared, so an aliased reference is tagged and + // forced nullable identically to a bare one. "ins" is an INSERT ... ON CONFLICT DO UPDATE // RETURNING WITH (OLD AS o) id, o.val AS oldval -- OLD is NULL exactly when the row was // freshly inserted -- and "m" merely passes ins.oldval through. PostgreSQL 18, with an "a" // row with no pre-existing "it2" row, so the INSERT always takes the fresh-insert branch: @@ -5101,11 +5149,11 @@ class QueryAnalysisTest { assumeTrue(pgVersion.substringBefore('.').toInt() >= 17, "merge_action() requires PostgreSQL 17+") // "m" (declared first) forward-references "pre" (declared after it) under WITH RECURSIVE, // which makes every sibling name visible to every other body regardless of declaration - // order. At e4679ff, this shape silently typed "bval" NOT NULL — the referencesAnyName - // sibling check (and its only trigger point) did not exist yet, so the stub path had - // nothing to force it nullable with, despite "pre"'s own LEFT JOIN null-extending it - // exactly as in the plain-WITH sibling test above. With an "a" row with no matching "b" - // row: act = 'UPDATE', bval = NULL, id = 1. + // order. The whole WITH RECURSIVE clause is one node tree PostgreSQL parses and resolves + // inside `BEGIN ATOMIC`, so "m"'s reference to "pre" resolves the same way whether "pre" is + // declared before or after it, and "pre"'s own LEFT JOIN null-extends "bval" regardless of + // declaration order. With an "a" row with no matching "b" row: act = 'UPDATE', bval = NULL, + // id = 1. val query = analyzeWithSchema( """ CREATE TABLE tgt (id INT PRIMARY KEY, tval TEXT NOT NULL); @@ -5133,12 +5181,11 @@ class QueryAnalysisTest { fun `DELETE RETURNING OLD and NEW both report nullable — NEW is always null for a deleted row`() { assumeTrue(pgVersion.substringBefore('.').toInt() >= 18, "RETURNING OLD requires PostgreSQL 18+") // PostgreSQL 18's RETURNING OLD/NEW: for a DELETE, OLD is the deleted row (always present) - // and NEW does not exist (always NULL). Neither the join-preserving conversion (OLD/NEW - // are not valid range variables outside RETURNING, so the converted SELECT fails to - // prepare) nor plain metadata (which reflects base-table attnotnull, oblivious to OLD/NEW's - // conditional existence) can see this — the safety net now forces both nullable whenever a - // body's RETURNING references OLD./NEW., regardless of join structure. On real Postgres, - // OLD.name = 'orig', NEW.name = NULL. + // and NEW does not exist (always NULL). `NodeTreeNullabilityAnalyzer.isNonNull` forces + // every OLD-tagged `Var` nullable unconditionally, and forces a NEW-tagged `Var` nullable + // too whenever `forceNewNullable` is set — true for a DELETE (see that constructor + // parameter's KDoc) — so both are reported nullable regardless of any join structure in the + // body. On real Postgres, OLD.name = 'orig', NEW.name = NULL. val query = analyzeWithSchema( "CREATE TABLE t (id INT NOT NULL, name TEXT NOT NULL)", """ @@ -5154,17 +5201,14 @@ class QueryAnalysisTest { } @Test - fun `INSERT ON CONFLICT RETURNING OLD-col is nullable even though INSERT skips the join-based net`() { + fun `INSERT ON CONFLICT RETURNING OLD-col is nullable independent of any join in the statement`() { assumeTrue(pgVersion.substringBefore('.').toInt() >= 18, "RETURNING OLD requires PostgreSQL 18+") - // This alone does not demonstrate the OLD/NEW safety-net fix: checked directly against the - // driver (PreparedStatement.getMetaData()), PostgreSQL's own metadata already reports - // OLD.bval as nullable for this exact shape, with no forceAllNullable involved — so this - // body would pass even without referencesOldOrNew. What this does confirm is that the two - // forceAllNullable triggers are independent: this body is an INSERT (per isInsertBody, - // excluded from the join-based trigger) with no join at all, and still correctly ends up - // nullable — proving isInsertBody's exclusion doesn't also (incorrectly) suppress the - // OLD/NEW trigger. The DELETE test below, where raw PostgreSQL metadata is wrong without - // the fix, is the demonstrative case. Ground truth for OLD.bval's real nullability: with + // OLD.bval is tagged by `:varreturningtype` regardless of statement kind, so the blanket + // OLD-forcing rule (`NodeTreeNullabilityAnalyzer.isNonNull`, + // `PgNodeExpression.Var.returningType`) reports it nullable here even though this body is a + // plain INSERT with no join at all — proving the OLD/NEW rule and the `:varnullingrels` + // join check are two independent mechanisms, not one that only fires when a join is also + // present. Ground truth for OLD.bval's real nullability: with // an existing row (a genuine conflict), OLD.bval = 'orig'; with no conflict (a fresh // insert), OLD.bval = NULL — so across possible executions the column is genuinely // nullable, not merely over-approximated. @@ -5204,9 +5248,10 @@ class QueryAnalysisTest { @Test fun `chained data-modifying CTE followed by SELECT CTE with LEFT JOIN referencing it`() { - // Regression guard: a non-DML CTE body must be kept verbatim (not stubbed) when the - // query is transformed for view creation, because a stub built from base-table - // `attnotnull` cannot reproduce nullability induced by a LEFT JOIN inside the CTE body. + // "j" is a plain SELECT CTE (not data-modifying) sitting alongside a data-modifying + // sibling ("ins") in the same WITH clause. Both are parsed as part of the same node tree + // regardless of what the sibling CTE contains, so "j"'s own LEFT JOIN carries its real + // `:varnullingrels` and "label" is correctly reported nullable. val query = analyzeWithSchema( """ CREATE TABLE t (id SERIAL NOT NULL, name TEXT NOT NULL); @@ -5229,9 +5274,9 @@ class QueryAnalysisTest { @Test fun `data-modifying CTE alongside unrelated SELECT CTE with LEFT JOIN`() { - // Same regression guard as above, without chaining: the SELECT CTE with the LEFT JOIN - // does not reference the data-modifying CTE at all, but the presence of DML anywhere - // in the query still triggers the view-creation transform for the whole statement. + // Same shape as above, without chaining: "j" (the SELECT CTE with the LEFT JOIN) does not + // reference the data-modifying CTE "ins" at all, confirming an unrelated data-modifying + // sibling elsewhere in the same WITH clause changes nothing about how "j" is analyzed. val query = analyzeWithSchema( """ CREATE TABLE t (id SERIAL NOT NULL, name TEXT NOT NULL); @@ -5255,9 +5300,9 @@ class QueryAnalysisTest { @Test fun `parenthesized SELECT CTE body with LEFT JOIN referencing chained DML CTE`() { - // Regression guard: a CTE body may itself be parenthesized (e.g. `AS ((SELECT ...))`). - // The leading-keyword check must skip past the extra `(` rather than misclassifying - // this SELECT body as data-modifying and stubbing away its LEFT JOIN. + // A CTE body may itself be parenthesized (e.g. `AS ((SELECT ...))`) — ordinary SQL syntax + // PostgreSQL's own parser strips while building the node tree this analysis reads, so the + // extra parentheses change nothing about how "j"'s LEFT JOIN is analyzed. val query = analyzeWithSchema( """ CREATE TABLE t (id SERIAL NOT NULL, name TEXT NOT NULL); @@ -5280,7 +5325,9 @@ class QueryAnalysisTest { @Test fun `parenthesized SELECT CTE body with leading block comment before the parenthesis`() { - // Same regression guard as above, with a block comment between "AS (" and the extra "(". + // Same shape as above, with a block comment between "AS (" and the extra "(" — again + // ordinary syntax PostgreSQL's own parser handles before this analysis ever sees the node + // tree. val query = analyzeWithSchema( """ CREATE TABLE t (id SERIAL NOT NULL, name TEXT NOT NULL); @@ -5301,8 +5348,10 @@ class QueryAnalysisTest { @Test fun `parenthesized UNION ALL CTE body remains non-null when both branches are non-null`() { - // Regression guard: a parenthesized UNION ALL body must also be kept verbatim (not - // stubbed as data-modifying), so its true non-null result is preserved. + // A parenthesized UNION ALL body is likewise ordinary syntax PostgreSQL's own parser + // resolves before this analysis ever sees the node tree; both branches are genuinely NOT + // NULL, and `ColumnNullabilityAnalyzer.analyzeSetOperationBranches` OR-combines them to the + // same NOT NULL answer. val query = analyzeWithSchema( """ CREATE TABLE t (id SERIAL NOT NULL, name TEXT NOT NULL); @@ -5324,9 +5373,9 @@ class QueryAnalysisTest { @Test fun `SELECT CTE body with nested block comment before it is kept verbatim`() { - // Regression guard: Postgres block comments nest (`/* a /* b */ */` is one comment), so - // the leading-keyword check must skip past the whole nested comment rather than stopping - // at the first "*/" and misclassifying this SELECT body as data-modifying. + // Postgres block comments nest (`/* a /* b */ */` is one comment) — again ordinary syntax + // PostgreSQL's own parser handles before this analysis ever sees the node tree, so "j"'s own + // LEFT JOIN is analyzed the same way regardless of what precedes it. val query = analyzeWithSchema( """ CREATE TABLE t (id SERIAL NOT NULL, name TEXT NOT NULL); @@ -5370,15 +5419,11 @@ class QueryAnalysisTest { fun `RETURNING item with no usable name is resolved by the outer query`() { // A RETURNING item that is neither a plain column reference nor a simple cast (here, // string concatenation) has no name of its own, so PostgreSQL reports it as the literal - // "?column?" (confirmed via psql \gdesc) — not a valid bare identifier at all. Before the - // fix, tryPrepareStub emitted it unquoted ("AS ?column?"), a syntax error in the stub - // SELECT, which failed CREATE VIEW the same way the mixed-case-alias test below's shape - // did. Deliberately concatenates the nullable "name" column (rather than a literal like - // "RETURNING 1", which is always non-null and would pass either way, masking the bug the - // same way "id" did in the test above) so the CREATE-VIEW failure's top-level - // analyzeUnconvertibleDml fallback (queryColumnNullability's last resort when - // analyzeViaTemporaryView on the transformed SQL itself throws) is observable — before - // this fix, that fallback wrongly asserted NOT NULL here regardless of truth. + // "?column?" (confirmed via psql \gdesc) — not a valid bare identifier at all, yet the + // outer query's unqualified "SELECT *" still resolves it by ordinal position through the + // CTE's own target list, not by name. "name" has no NOT NULL constraint, so `name || 'x'` + // is genuinely nullable, and `NodeTreeNullabilityAnalyzer`'s ordinary expression handling + // reports it as such regardless of what name PostgreSQL assigns the column. val query = analyzeWithSchema( "CREATE TABLE t (id SERIAL NOT NULL, name TEXT)", """ @@ -5395,19 +5440,13 @@ class QueryAnalysisTest { @Test fun `quoted mixed-case RETURNING alias in a data-modifying CTE body is resolved by the outer query`() { - // "ins"'s body is a plain INSERT, so it never reaches convertDmlCteBodyToSelect (whose - // join-preserving conversion is limited to UPDATE/DELETE/MERGE) and stays on the - // tryPrepareStub path. ResultSetMetaData.getColumnName reports the RETURNING alias exactly - // as declared, "myId", but before the fix tryPrepareStub emitted it unquoted ("AS myId"), - // which PostgreSQL folds to lowercase "myid" when building the stub SELECT used for - // CREATE VIEW. The outer query's quoted reference to ins."myId" then fails to resolve - // against the stub ("column ins.myId does not exist" — confirmed via psql). Inside - // queryColumnNullability, that SQLException is caught and degraded to - // analyzeUnconvertibleDml's fallback — before this fix, that fallback asserted every - // column NOT NULL regardless of truth — so "name" is nullable in the schema (no NOT NULL - // constraint), but before the fix this test wrongly reports it NOT NULL. Deliberately uses - // a nullable source column (not id, which is NOT NULL and would pass either way, masking - // the bug) so the wrong fallback is actually observable as an assertion failure. + // "ins"'s body is a plain INSERT with a quoted, mixed-case RETURNING alias ("myId"). The + // whole statement is handed to PostgreSQL whole inside `BEGIN ATOMIC` + // (`ColumnNullabilityAnalyzer.queryColumnNullabilityViaProsqlbody`); nothing re-composes SQL + // text or re-quotes an alias, so PostgreSQL's own parser resolves the outer query's quoted + // reference to `ins."myId"` regardless of whether "ins"'s body is an INSERT or an + // UPDATE/DELETE/MERGE. "name" has no NOT NULL constraint in the schema, so it is correctly + // reported nullable. val query = analyzeWithSchema( "CREATE TABLE t (id SERIAL NOT NULL, name TEXT)", """ @@ -5424,13 +5463,11 @@ class QueryAnalysisTest { @Test fun `RETURNING alias with an embedded double quote in a data-modifying CTE body is resolved by the outer query`() { - // Same failure mode as the test above, but the alias itself contains a literal double - // quote (written "my""Id" in SQL, an escaped quote inside a quoted identifier, so the - // real column name is my"Id). tryPrepareStub must double the embedded quote when - // re-quoting the alias for the stub SELECT ("AS \"my\"\"Id\""); emitting only a single - // doubled quote or none at all would produce invalid SQL or fold/mismatch the name, and - // the outer query's reference would fail to resolve the same way as the test above — - // degrading to the same wrong-NOT-NULL fallback described there. + // Same shape as the test above, but the alias itself contains a literal double quote + // (written "my""Id" in SQL, an escaped quote inside a quoted identifier, so the real column + // name is my"Id). The statement is handed to PostgreSQL whole; nothing here re-quotes or + // re-escapes the alias, so the escaped quote is preserved exactly as PostgreSQL's own parser + // reads it, and the outer query's matching reference resolves the same way. val query = analyzeWithSchema( "CREATE TABLE t (id SERIAL NOT NULL, name TEXT)", """ @@ -5447,11 +5484,12 @@ class QueryAnalysisTest { @Test fun `a CTE body's own local WITH shadowing a sibling of the same name resolves against the local body`() { - // buildInnerCteNotNull resolved a direct :rtable reference to a CTE against - // previouslyResolved (sibling CTEs) with no ctelevelsup check, so b's own local "c" (over - // nullable w.v) was shadowed by the outer sibling "c" (over NOT NULL u.v) and this reported - // notNull=true. PostgreSQL 18: returns null once w has a NULL row, since b's own body - // reads its own local c, not the outer one. + // `QueryBlockScope.isSourceColumnNotNull`'s CTE branch resolves a direct `:rtable` + // reference to a CTE by checking the reference's own `:ctelevelsup`: `0` selects `ownCtes` + // (the CTE declared directly in "b"'s own nested `WITH`), anything greater selects + // `enclosingCtes` — so b's own local "c" (over nullable w.v) is never shadowed by the outer + // sibling "c" (over NOT NULL u.v). PostgreSQL 18: returns null once w has a NULL row, since + // b's own body reads its own local c, not the outer one. val query = analyzeWithSchema( "CREATE TABLE u (v TEXT NOT NULL); CREATE TABLE w (v TEXT)", """ @@ -5466,10 +5504,11 @@ class QueryAnalysisTest { @Test fun `an ANY sublink inside a CTE body resolves a shadowing local WITH, not the outer sibling`() { // Same ctelevelsup hazard as the direct-reference test above, but reached through a SubLink - // inside b's own body instead of a plain target-list reference — this is what - // buildAnalyzer's `resolvedCtes = ownResolvedCtes` (not previouslyResolved) at - // buildCteBodyAnalyzer's call site protects. PostgreSQL 18: returns null once w has a NULL - // row and t.a matches no non-null row of b's own local sib. + // inside b's own body instead of a plain target-list reference — `buildAnalyzer`'s + // `resolvedCtes` parameter is threaded from `QueryBlockScope.ownCtes` specifically so a + // `SubLink`'s own subselect resolves against the query block's own nested `WITH`, not an + // outer sibling of the same name. PostgreSQL 18: returns null once w has a NULL row and t.a + // matches no non-null row of b's own local sib. val query = analyzeWithSchema( """ CREATE TABLE t (id INT NOT NULL, a TEXT NOT NULL); @@ -5485,6 +5524,20 @@ class QueryAnalysisTest { assertThat(query.columns).hasSize(1) assertThat(query.columns[0].notNull).isFalse() } + + @Test + fun `a CTE referenced only from inside a FROM subquery resolves via its ctelevelsup 1 reference`() { + // c is never referenced in the outer query's own :rtable at all — only inside the derived + // table s's own :rtable, where the reference carries :ctelevelsup 1 pointing back up to the + // outer WITH clause, a shape no earlier test in this class covers. PostgreSQL 18: returns + // id NOT NULL, since t.id is NOT NULL and c and s are both plain passthroughs of it. + val query = analyzeWithSchema( + "CREATE TABLE t (id INT NOT NULL)", + "WITH c AS (SELECT id FROM t) SELECT s.id FROM (SELECT id FROM c) s", + ) + assertThat(query.columns).hasSize(1) + assertThat(query.columns[0].notNull).isTrue() + } } @Nested @@ -5644,7 +5697,7 @@ class QueryAnalysisTest { // RETURNING * on a MERGE expands to both relations' columns — this exact statement returns // 4 columns (s.id, s.name, t.id, t.name), all genuinely NOT NULL here // (the source is a fixed-literal derived table, never actually absent or null). But - // PgCatalogLoader.mergeAbsentVarnos only attributes a MERGE's join to a source relation + // ColumnNullabilityAnalyzer.mergeAbsentVarnos only attributes a MERGE's join to a source relation // that is itself a plain base table (an :rtable entry with rtekind 0) — a subquery/VALUES // source has no real relation OID or name EXPLAIN's plan can be correlated against, so this // shape falls back to reporting every column nullable rather than guessing. This is the @@ -5669,26 +5722,16 @@ class QueryAnalysisTest { @Test fun `top-level MERGE RETURNING merge_action() does not abort generation`() { assumeTrue(pgVersion.substringBefore('.').toInt() >= 17, "merge_action() requires PostgreSQL 17+") - // convertDmlToSelect splices merge_action() into a plain SELECT verbatim, where it is not - // valid PostgreSQL (merge_action() only works inside MERGE's own RETURNING) — so the - // converted SELECT fails to prepare. At d1153f3, Phase 2 (the top-level, non-CTE conversion - // path) had no validation gate at all: the bad SELECT reached CREATE VIEW, and the - // resulting SQLException was thrown from inside queryColumnNullability's own catch block, - // escaping uncaught and aborting the whole build on SQL PostgreSQL itself accepts fine. On - // real Postgres, with a matched row and no WHEN NOT MATCHED branch — every returned row - // genuinely has a target and source row present: act = 'UPDATE', aval = 'a1', id = 1 — all - // three genuinely NOT NULL. - // - // "aval" and "id" are NOT NULL via honestly-read ResultSetMetaData (a simple column - // reference tracing to its source column's attnotnull). "act" (a bare function call) reports - // `columnNullableUnknown`; probeUnknownColumnNullability is attempted but cannot resolve it: - // merge_action() is only valid inside a real MERGE's own RETURNING list, so the plain - // `SELECT merge_action() AS act, a.aval, tgt.id FROM tgt` probe this builds fails to prepare - // (both because merge_action() itself is invalid there, and because "a" isn't in that - // probe's FROM list at all) — the probe returns `null` and analyzeUnconvertibleDml falls - // back to its own nullable default for "act": nullability analysis must be correct or - // silent, and this file cannot prove merge_action() is non-null here even though it always - // is in practice. + // The USING source is the base table "a", so the MERGE is analyzed rather than bailed on, + // and "aval" and "id" are plain column references reported NOT NULL from their own catalog + // constraints. "act" is a bare `merge_action()` call, which PostgreSQL emits as a + // `MERGESUPPORTFUNC` node; `PgNodeTreeParser.parseExpression` has no case for that node kind, + // so it becomes a `PgNodeExpression.Unknown` and is reported nullable. The node carries no + // `:funcid`, so the safe-list and strictness legs are never consulted at all. On real Postgres, with a + // matched row and no WHEN NOT MATCHED branch — every returned row genuinely has a target and + // source row present: act = 'UPDATE', aval = 'a1', id = 1 — all three genuinely NOT NULL, + // but this test only pins that "act" (the one column whose true non-null-ness this analysis + // cannot prove) is reported nullable, the safe direction. val query = analyzeWithSchema( """ CREATE TABLE tgt (id INT PRIMARY KEY, tval TEXT NOT NULL); @@ -5709,13 +5752,13 @@ class QueryAnalysisTest { @Test fun `top-level MERGE RETURNING merge_action-comma-star does not abort generation`() { assumeTrue(pgVersion.substringBefore('.').toInt() >= 17, "merge_action() requires PostgreSQL 17+") - // The four plain column references (both "id"s, "aval", "tval") are NOT NULL via - // honestly-read ResultSetMetaData. "merge_action" (a bare function call, - // `columnNullableUnknown`) cannot be proven: star-expansion in probeUnknownColumnNullability - // only knows the MERGE's own target ("tgt"), not its "USING a" source, so the expanded item - // count (3: merge_action() + tgt's own 2 columns) doesn't match the real 5-column result — - // the probe correctly bails rather than trust a mapping it can't verify, falling back to - // "act"'s nullable default. + // The four plain column references (both "id"s, "aval", "tval") are correctly reported NOT + // NULL from their own catalog constraints, and PostgreSQL's own star expansion + // (`PgNodeTreeParser.parseReturningList`) supplies all four regardless of how many relations + // the MERGE reads from. "merge_action()" is, as in the test above, a `MERGESUPPORTFUNC` + // node that `PgNodeTreeParser.parseExpression` does not recognize, so it becomes a + // `PgNodeExpression.Unknown` and is reported nullable — the same answer, for the same + // reason, as the single-column case above. val query = analyzeWithSchema( """ CREATE TABLE tgt (id INT PRIMARY KEY, tval TEXT NOT NULL); @@ -5738,30 +5781,16 @@ class QueryAnalysisTest { @Test fun `top-level MERGE RETURNING OLD-col does not abort generation`() { assumeTrue(pgVersion.substringBefore('.').toInt() >= 18, "RETURNING OLD requires PostgreSQL 18+") - // Same crash as above, OLD-reference variant (also not valid outside RETURNING, same - // failure mode as merge_action()) — rejected conversion falls through to - // analyzeUnconvertibleDml. On real Postgres, with a matched-row-only MERGE, so every - // returned row's OLD is the pre-existing target row, always present: OLD.tval = 'x', - // genuinely NOT NULL for this exact shape. Before this fix, this happened to be reported - // correctly (NOT NULL) only by coincidence, via the old fallback that asserted every column - // NOT NULL unconditionally. analyzeUnconvertibleDml now applies the same per-column OLD/NEW - // forcing the CTE-body stub path always has (see `UPDATE RETURNING OLD-col alongside the - // target's own column stays NOT NULL for the target column` above for the CTE-wrapped - // precedent), which forces every OLD/NEW-referencing column nullable by design regardless - // of whether a specific MERGE shape happens to make it always present — an accepted, - // deliberate loss of precision in the safe direction, not a regression. - // - // This assertion cannot be restored to main's `isTrue()`: this column's own nullability is - // driven entirely by the per-column OLD/NEW forcing above, never by - // `metadata.isNullable`/`columnNullableUnknown` (there is no non-OLD/NEW column here at - // all), so it is untouched by, and independent of, the `columnNullableUnknown` handling fix - // (see the `merge_action()` tests above). Reverting it to NOT NULL would mean removing the - // OLD/NEW forcing for this shape specifically while keeping it for `WHEN NOT MATCHED THEN - // INSERT ... RETURNING OLD.tval` (the "freshly-inserted row" test below), which the text - // scan this predicate runs on cannot distinguish — both are `RETURNING OLD.tval` on a - // single-item list; only the `WHEN` branches differ, and no static scan tells them apart. - // Keeping the over-approximation for both is the same accepted tradeoff already documented - // above. + // Same OLD reference as the CTE-wrapped precedent above (`RETURNING OLD-col is nullable by + // rule while a separate LEFT JOIN column is nullable by its own real join`), but as a bare + // top-level statement instead of one wrapped in a CTE — the same `Var.returningType` + // tagging and the same blanket OLD-forcing rule (`NodeTreeNullabilityAnalyzer.isNonNull`) + // apply regardless of whether the statement sits inside a CTE, so this pins that being + // top-level changes nothing. On real Postgres, with a matched-row-only MERGE, every returned + // row's OLD is the pre-existing target row, always present: OLD.tval = 'x', genuinely NOT + // NULL for this exact shape — but the rule is deliberately statement-shape-agnostic (see + // `PgNodeExpression.Var.returningType`'s KDoc), so this column is still reported nullable + // here, the safe direction. val query = analyzeWithSchema( """ CREATE TABLE tgt (id INT PRIMARY KEY, tval TEXT NOT NULL); @@ -5780,11 +5809,11 @@ class QueryAnalysisTest { @Test fun `MERGE INTO with WHEN NOT MATCHED THEN INSERT RETURNING OLD-col is nullable for a freshly-inserted row`() { assumeTrue(pgVersion.substringBefore('.').toInt() >= 18, "RETURNING OLD requires PostgreSQL 18+") - // A freshly INSERTed row via MERGE has no prior row, so OLD.tval is genuinely NULL — - // before the fix, the top-level no-join-structure fallback - // (analyzeUnconvertibleDml's predecessor) asserted it NOT NULL unconditionally. On real - // Postgres, with a is INSERT INTO a VALUES (1, 'a1'), tgt starts empty, so a.id has no - // matching tgt row and WHEN NOT MATCHED fires: oldv = NULL. + // A freshly INSERTed row via MERGE has no prior row, so OLD.tval is genuinely NULL, and the + // blanket OLD-forcing rule (`NodeTreeNullabilityAnalyzer.isNonNull`, + // `PgNodeExpression.Var.returningType`) reports it nullable regardless of which MERGE action + // produced the row. On real Postgres, with a is INSERT INTO a VALUES (1, 'a1'), tgt starts + // empty, so a.id has no matching tgt row and WHEN NOT MATCHED fires: oldv = NULL. val query = analyzeWithSchema( """ CREATE TABLE tgt (id INT PRIMARY KEY, tval TEXT NOT NULL); @@ -5841,15 +5870,14 @@ class QueryAnalysisTest { @Test fun `top-level MERGE RETURNING merge_action() alongside a LEFT JOIN in USING forces every column nullable`() { assumeTrue(pgVersion.substringBefore('.').toInt() >= 17, "merge_action() requires PostgreSQL 17+") - // merge_action() forces Phase 2's conversion to be rejected (not valid outside MERGE's own - // RETURNING), so this falls to analyzeUnconvertibleDml — which now detects the LEFT JOIN - // nested in the USING subquery via the same null-extending-construct trigger the CTE-body - // stub path already has, forcing every column nullable, "act" included — the same accepted - // over-approximation the CTE-wrapped equivalent test above (`MERGE with a LEFT JOIN nested - // in its USING subquery reports the joined column nullable`) documents, now reached for a - // bare top-level statement instead of one wrapped in a CTE. On real Postgres, with b - // having no row matching a: act = 'UPDATE', bval = NULL (genuinely nullable — the LEFT - // JOIN's real effect). + // Bare top-level counterpart of the CTE-wrapped `MERGE with a LEFT JOIN nested in its USING + // subquery reports the joined column nullable` test above, and it bails for the same reason: + // the USING source is a subquery, so `mergeSourceRelationNameCandidates` returns `null` and + // `queryColumnNullabilityViaProsqlbody` gives up, leaving `queryColumnNullability` to report + // every column nullable. Both "bval" and "act" are nullable by that fallback, not by any + // per-column reasoning about the join or the function. On real Postgres, with b having no + // row matching a: act = 'UPDATE', bval = NULL (genuinely nullable — the LEFT JOIN's real + // effect). val query = analyzeWithSchema( """ CREATE TABLE tgt (id INT PRIMARY KEY, tval TEXT NOT NULL); @@ -5869,12 +5897,10 @@ class QueryAnalysisTest { @Test fun `top-level DELETE RETURNING reports a nullable column as nullable, not NOT NULL`() { - // The everyday, no-OLD-NEW-MERGE case — a plain top-level DELETE with no FROM/USING clause - // has no join structure for convertDmlToSelect to convert, so it goes - // straight to analyzeUnconvertibleDml, which now reads real ResultSetMetaData.isNullable - // instead of discarding it. "note" has no NOT NULL constraint, so it is genuinely nullable; - // "id" and "name" are declared NOT NULL and stay that way — this is not "mark everything - // nullable", only "consult the metadata this fallback had all along". + // The everyday case — a plain top-level DELETE with no FROM/USING clause reads its + // RETURNING columns as ordinary `Var`s against the target table's own catalog constraints + // (`ColumnNullabilityAnalyzer.isColumnNotNull`): "note" has no NOT NULL constraint, so it is + // genuinely nullable; "id" and "name" are declared NOT NULL and stay that way. val query = analyzeWithSchema( "CREATE TABLE t (id INT PRIMARY KEY, name TEXT NOT NULL, note TEXT)", "DELETE FROM t WHERE id = ? RETURNING id, name, note", @@ -5888,7 +5914,8 @@ class QueryAnalysisTest { @Test fun `top-level UPDATE RETURNING reports a nullable column as nullable, not NOT NULL`() { // UPDATE equivalent of the DELETE shape above — a plain top-level UPDATE with no FROM - // clause has no join structure to convert either, so it hits the same fallback. + // clause reads its RETURNING columns the same way, against the target table's own catalog + // constraints. val query = analyzeWithSchema( "CREATE TABLE t (id INT PRIMARY KEY, name TEXT NOT NULL, note TEXT)", "UPDATE t SET name = ? WHERE id = ? RETURNING id, name, note", @@ -5901,14 +5928,12 @@ class QueryAnalysisTest { @Test fun `top-level UPDATE with a LEFT JOIN only inside a WHERE subquery does not force RETURNING nullable`() { - // The null-extending-construct arm used to scan the whole statement's text for an outer - // join, not just the clause whose join structure can actually reach RETURNING — so a LEFT - // JOIN sitting inside an unrelated `WHERE ... IN (subquery)` (which only narrows which rows - // the UPDATE touches, and cannot null-extend anything in RETURNING) fabricated nullable for - // both columns. PostgreSQL 18, with a matching row existing so the WHERE filter passes; - // t.id is even the PRIMARY KEY: id and name are genuinely NOT NULL. - // dmlSourceClauseRegion returns null here (no top-level FROM clause on this UPDATE at all), - // so the join arm is forced off rather than scanning the WHERE subquery's own LEFT JOIN. + // A LEFT JOIN sitting inside an unrelated `WHERE ... IN (subquery)` only narrows which rows + // the UPDATE touches and cannot null-extend anything in the outer RETURNING list. "t.id" and + // "t.name" are ordinary `Var`s against the UPDATE's own target relation, so PostgreSQL's own + // planner never attaches `:varnullingrels` to them regardless of what the WHERE subquery + // contains. PostgreSQL 18, with a matching row existing so the WHERE filter passes; t.id is + // even the PRIMARY KEY: id and name are genuinely NOT NULL. val query = analyzeWithSchema( """ CREATE TABLE t (id INT PRIMARY KEY, name TEXT NOT NULL); @@ -5929,8 +5954,8 @@ class QueryAnalysisTest { @Test fun `top-level DELETE with a LEFT JOIN only inside a WHERE subquery does not force RETURNING nullable`() { // DELETE equivalent of the UPDATE case above — a LEFT JOIN inside a `WHERE ... IN - // (subquery)` cannot null-extend a plain DELETE's RETURNING list either, and this DELETE - // has no top-level USING clause at all for dmlSourceClauseRegion to scope to. + // (subquery)` cannot null-extend a plain DELETE's RETURNING list either, for the same + // `:varnullingrels` reason. val query = analyzeWithSchema( """ CREATE TABLE t (id INT PRIMARY KEY, name TEXT NOT NULL); @@ -5951,22 +5976,16 @@ class QueryAnalysisTest { @Test fun `top-level UPDATE FROM with OLD-col still finds the real LEFT JOIN in its scoped region`() { assumeTrue(pgVersion.substringBefore('.').toInt() >= 18, "RETURNING OLD requires PostgreSQL 18+") - // A bare `UPDATE ... FROM a LEFT JOIN b ... RETURNING t.name, b.bval` converts and validates - // successfully via the join-aware node-tree analyzer, never reaching - // analyzeUnconvertibleDml at all — so it cannot exercise the scoped fallback path by itself. - // Adding a RETURNING OLD.col forces the structural conversion to be rejected (OLD is not a - // valid range variable in the converted SELECT), landing on analyzeUnconvertibleDml — the - // top-level counterpart of `stub path forces every column nullable when RETURNING OLD-col - // accompanies a real LEFT JOIN` above. dmlSourceClauseRegion scopes the join scan to this - // UPDATE's own FROM ... WHERE region, which does contain the real LEFT JOIN, so bval (and, - // by the same accepted over-approximation as the CTE case, oldname and name too) are forced - // nullable — proving the scoped region still finds a join that genuinely belongs to the - // source clause, not merely refusing to force anything at all. - // prosqlbody reads the raw :targetList assignment for "name" directly (a literal 'x', - // untouched by either the OLD-forcing rule or the LEFT JOIN), so it correctly isolates - // "name" as NOT NULL, "oldname" as nullable (the blanket OLD-forcing rule), and "bval" as - // nullable (the genuine LEFT JOIN). On real Postgres, this exact UPDATE returns `name = - // 'x'`, never NULL, for a matching row. + // A bare `UPDATE ... FROM a LEFT JOIN b ... RETURNING t.name, b.bval` (no OLD/NEW + // reference) is reported correctly by the ordinary node-tree analysis on its own. Adding a + // RETURNING OLD.col changes nothing structural — the whole statement, LEFT JOIN included, is + // still one node tree — it only adds one more `Var` whose `:varreturningtype` tags it OLD, + // which the blanket OLD-forcing rule reports nullable independently of the join. prosqlbody + // reads the raw :targetList assignment for "name" directly (a literal 'x', untouched by + // either the OLD-forcing rule or the LEFT JOIN), so it correctly isolates "name" as NOT + // NULL, "oldname" as nullable (the blanket OLD-forcing rule), and "bval" as nullable (the + // genuine LEFT JOIN, via its own `:varnullingrels`). On real Postgres, this exact UPDATE + // returns `name = 'x'`, never NULL, for a matching row. val query = analyzeWithSchema( """ CREATE TABLE t (id INT NOT NULL, name TEXT NOT NULL); @@ -6010,16 +6029,11 @@ class QueryAnalysisTest { @Test fun `INSERT RETURNING a literal and a bare integer constant reports both NOT NULL`() { - // A literal or constant expression RETURNING item reports - // ResultSetMetaData.columnNullableUnknown (PostgreSQL cannot describe a literal's - // nullability any more precisely than that) — a literal is never NULL, so - // analyzeUnconvertibleDml must treat that as NOT NULL. The probe actually confirms this - // exactly, rather than merely defaulting to it: it builds `SELECT id, name, 'lit'::TEXT AS - // lbl, 1 AS one FROM t` and reads its real per-column nullability via the same node-tree - // analyzer a plain SELECT already uses, independently reporting both literal columns NOT - // NULL. "name" has no NOT NULL constraint and is left untouched by this rule (the probe's - // own answer for it is irrelevant), since it traces to a real column whose base-table - // attnotnull is already known (columnNoNulls), not unknown. + // `'lit'::TEXT` and `1` are each a `PgNodeExpression.Const`; + // `NodeTreeNullabilityAnalyzer.isNonNull`'s `Const` case is simply `!expression.isNull`, so a + // non-NULL literal is reported NOT NULL directly from the node tree, with no separate probe + // of any kind involved. "name" has no NOT NULL constraint and traces to a real column's own + // catalog answer, independent of the literal columns. val query = analyzeWithSchema( "CREATE TABLE t (id SERIAL PRIMARY KEY, name TEXT)", "INSERT INTO t (name) VALUES (?) RETURNING id, name, 'lit'::TEXT AS lbl, 1 AS one", @@ -6033,12 +6047,14 @@ class QueryAnalysisTest { } /** - * `PgCatalogLoader.analyzeUnconvertibleDml` treats `ResultSetMetaData.columnNullableUnknown` - * as NOT NULL — correct for a literal/constant (`1 AS one`), but wrong for an expression built - * over a genuinely nullable source column (`lower(note)`, where `note` has no NOT NULL - * constraint). These tests exercise `PgCatalogLoader.probeUnknownColumnNullability`, the - * supplementary probe that resolves such a column's real nullability instead of defaulting, - * and the gates that fall back to today's NOT NULL default when the probe cannot be trusted. + * A `RETURNING` expression is analyzed by tracing its own node tree back to its source column, + * not by treating an unresolvable `ResultSetMetaData` type as NOT NULL — so `lower(note)` + * reports nullable precisely because `note` itself has no `NOT NULL` constraint in the catalog, + * without this the expression would report NOT NULL wrongly. These tests exercise + * [ColumnNullabilityAnalyzer.queryColumnNullabilityViaProsqlbody] and + * [ColumnNullabilityAnalyzer.analyzeNodeTree], which resolve that answer directly from the + * parsed statement, plus the cases (a star item, a trailing line comment) that must not disturb + * that resolution. */ @Nested inner class UnknownColumnNullabilityProbe { @@ -6069,8 +6085,8 @@ class QueryAnalysisTest { @Test fun `DELETE RETURNING an expression over a NOT NULL column reports NOT NULL, proving exactness`() { - // The probe must not blanket-flip every columnNullableUnknown column to nullable — only a - // genuinely nullable source column should surface as nullable through it. + // Node-tree tracing must not blanket-report every RETURNING expression as nullable — only an + // expression built over a genuinely nullable source column should surface as nullable. val query = analyzeWithSchema( "CREATE TABLE t (id INT PRIMARY KEY, name TEXT NOT NULL)", "DELETE FROM t WHERE id = ? RETURNING lower(name)", @@ -6092,9 +6108,9 @@ class QueryAnalysisTest { @Test fun `a star item in RETURNING is expanded so the probe still runs and reports the real answer`() { // "note" has no NOT NULL constraint, so lower(note) genuinely can be NULL — a star item must - // not prevent the probe from proving that: probeUnknownColumnNullability expands "*" against - // "t"'s own catalog columns (id, note) before counting items, so the 2-item RETURNING list - // ("*", "lower(note)") correctly resolves to the real 3-column count and the probe runs. + // not prevent the node tree from proving that: PostgreSQL itself expands "*" into individual + // :targetList/:returningList entries ("id", "note", "lower(note)") during its own parse, so + // the 3-entry list is traced exactly as any explicit list would be. val query = analyzeWithSchema( "CREATE TABLE t (id INT PRIMARY KEY, note TEXT)", "DELETE FROM t WHERE id = ? RETURNING *, lower(note)", @@ -6115,10 +6131,11 @@ class QueryAnalysisTest { @Test fun `a trailing line comment on the RETURNING list does not swallow the probe's FROM clause`() { - // Regression: probeUnknownColumnNullability used to compose "SELECT $returningText FROM - // $target" on a single line. A trailing "--" comment with nothing after it on that same - // line (no line break of its own to stop at) swallowed " FROM t" into the comment, making - // the probe fail to prepare and silently degrade to today's NOT NULL default instead of the + // Regression: the prosqlbody wrapper composes "BEGIN ATOMIC $substitutedSql\n; END" (see + // `ColumnNullabilityAnalyzer.queryColumnNullabilityViaProsqlbody`). A trailing "--" + // comment with nothing after it on that same line (no line break of its own to stop at) + // would swallow "; END" into the comment if the newline before it were missing, making the + // probe function fail to create and silently degrade to the NOT NULL default instead of the // real (nullable) answer. val query = analyzeWithSchema( "CREATE TABLE t (id INT PRIMARY KEY, note TEXT)", @@ -6143,14 +6160,15 @@ class QueryAnalysisTest { } /** - * [PgCatalogLoader.probeUnknownColumnNullability]'s bare `SELECT FROM ` - * probe evaluates a `RETURNING` expression against the unmodified target table, so it cannot - * see a value the statement's own `SET` clause assigns — `UPDATE t SET note = 'x' RETURNING - * lower(note)` widened to nullable even though `note` can only ever be `'x'` in this result, - * because `note` has no `NOT NULL` constraint in the catalog. These tests exercise - * [PgCatalogLoader.buildUpdateSetAwareFromClause], which wraps the probe's target in a derived - * table carrying the `SET`-assigned expressions, plus every bail condition that must keep the - * bare, pre-fix column instead. + * A `RETURNING` expression read against a target column's general catalog constraint alone + * would report `UPDATE t SET note = 'x' RETURNING lower(note)` nullable, even though `note` can + * only ever be `'x'` in this result, because `note` itself has no `NOT NULL` constraint. These + * tests exercise [ColumnNullabilityAnalyzer.analyzeNodeTree]'s `:targetList`-to-`:returningList` + * substitution (lines 539-557): a `:returningList` `Var` on `(resultRelationVarno, attno)` is + * evaluated as the matching `:targetList` assigned expression instead, gated by + * `trustAssignedExpressions = '?' !in sql` (line 451) and + * [ColumnNullabilityAnalyzer.isSubstitutionSafeForRelation] (line 1260) — plus every bail + * condition that must keep the untrusted, general-constraint answer instead. */ @Nested inner class SetAssignmentAwareProbe { @@ -6231,16 +6249,14 @@ class QueryAnalysisTest { @Test fun `a system column the SET-aware derived table can't carry doesn't collapse a sibling column's real answer`() { - // The wrapped `FROM (SELECT ... FROM t) AS t` derived table has no `ctid` — PostgreSQL fails + // History: no SQL is re-composed today, so no prepare can fail here at all. `ctid`, a system + // column (negative attnum), is unconditionally treated NOT NULL by + // ColumnNullabilityAnalyzer.isColumnNotNull (line 621) and stays NOT NULL regardless. + // Historically, a derived table wrapping `FROM t` had no `ctid` column, so PostgreSQL failed // to prepare `SELECT ..., ctid FROM (SELECT ... FROM t) AS t` with "column \"ctid\" does not - // exist". `ctid` itself is reported NOT NULL directly by `ResultSetMetaData` on the raw - // `UPDATE ... RETURNING` statement — it never goes through the probe at all, at HEAD or - // here — so it stays NOT NULL regardless. What the wrapped probe's failure must not be - // allowed to do is collapse the separate, otherwise-resolvable `lower(note)` column's own - // answer down to the probe's NOT NULL default: without the bare-`FROM t` retry, the whole - // combined probe (covering every `RETURNING` item at once) fails to prepare because of - // `ctid` alone, silently breaking `lower(note)`'s nullability too even though nothing about - // `ctid` bears on it. + // exist" — and that failure was capable of collapsing the separate, otherwise-resolvable + // `lower(note)` column's own answer down to a NOT NULL default too, even though nothing + // about `ctid` bore on it. This test pins that the two columns' answers stay independent. val query = analyzeWithSchema(schema, "UPDATE t SET note = note || 'x' RETURNING lower(note) AS n, ctid") assertThat(query.columns).hasSize(2) assertThat(query.columns[0].notNull).isFalse() @@ -6249,9 +6265,13 @@ class QueryAnalysisTest { @Test fun `an untyped literal assigned to a jsonb column reports nullable rather than failing the whole probe`() { - // Splicing the untyped literal `'{"a":1}'` into the derived table's column list degrades it - // to `text` there, so `data -> 'zzz'` (the `->` jsonb operator) fails to prepare against the - // wrapped probe. The bare-target retry sees the real `jsonb` column instead and succeeds. + // History: splicing an untyped literal into a re-composed derived table's column list used + // to degrade its type to `text`, so `data -> 'zzz'` (the `->` jsonb operator) could fail to + // prepare against the wrong-typed target and mask this column's real answer behind a NOT + // NULL default. No SQL is re-composed today — the real UPDATE statement's own node tree + // already types `data` as `jsonb` (see ColumnNullabilityAnalyzer.analyzeNodeTree, lines + // 539-557), so `data -> 'zzz'` resolves the real jsonb `->` operator and reports nullable + // because the key may be absent, not because of any fallback. val query = analyzeWithSchema(schema, "UPDATE t SET data = '{\"a\":1}' RETURNING data -> 'zzz' AS v") assertThat(query.columns).hasSize(1) assertThat(query.columns[0].notNull).isFalse() @@ -6279,18 +6299,19 @@ class QueryAnalysisTest { } /** - * [SetAssignmentAwareProbe]'s substitution splices the `SET` right-hand side into the derived - * table's column list verbatim, with no cast to the column's own declared type. An untyped - * literal (`'empty'`) spliced bare is typed `text` by PostgreSQL inside the derived table — a - * different type than the real column's — which can resolve a `RETURNING` function call - * against a completely different, sometimes safe-listed, overload than the real statement would - * ever use: `lower(text)` is safe-listed, `lower(anyrange)` is not, and `UPDATE t SET r = - * 'empty' RETURNING lower(r)` resolves the latter against the real column but (pre-fix) the - * former against the bare-text derived table, silently reporting NOT NULL for a - * value that is actually `NULL` at runtime. [PgCatalogLoader.buildUpdateSetAwareFromClause] now - * casts every substituted expression to the column's own declared type — via - * [PgCatalogLoader.lookupDeclaredColumnTypes]'s `format_type(atttypid, atttypmod)` — so the - * derived table's column type is identical to the real one and resolves the identical overload. + * History: a derived-table substitution used to splice the `SET` right-hand side into a + * re-composed statement with no cast to the column's own declared type. An untyped literal + * (`'empty'`) spliced bare was typed `text` there — a different type than the real column's — + * which resolved `RETURNING`'s function call against a different overload than the real + * statement used: `lower(text)` is safe-listed, `lower(anyrange)` is not, and `UPDATE t SET r = + * 'empty' RETURNING lower(r)` resolved the latter against the real column but the former against + * the re-typed derived table, silently reporting NOT NULL for a value that is actually `NULL` at + * runtime. + * + * This hazard is now structurally impossible: no statement is ever re-typed. `lower(r)`'s + * `funcid` was already resolved by PostgreSQL against the real, correctly-typed column when it + * parsed the actual statement, and Norm only reads that already-resolved `funcid` from the node + * tree — there is no second type-resolution pass left for an untyped literal to derail. */ @Nested inner class SetAssignmentAwareProbeDeclaredTypeCast { @@ -6327,10 +6348,10 @@ class QueryAnalysisTest { @Test fun `control - a non-null text literal assigned to a nullable TEXT column still reports NOT NULL`() { - // The declared type of a plain TEXT column is "text" with no typmod, exactly what the - // untyped literal already defaulted to before this fix — so adding the cast must not change - // this answer. Same case as SetAssignmentAwareProbe's own first test, re-asserted here next - // to the cast-introducing fix as an explicit before/after control. + // History: pins the ordinary case unaffected by the derived-table/cast mechanism that used + // to exist for overload safety on non-text columns — same case as SetAssignmentAwareProbe's + // own first test, re-asserted here next to that mechanism's other regression tests as an + // explicit control. val query = analyzeWithSchema( "CREATE TABLE t (id INT PRIMARY KEY, note TEXT)", "UPDATE t SET note = 'x' WHERE id = 1 RETURNING lower(note) AS n", @@ -6351,12 +6372,11 @@ class QueryAnalysisTest { @Test fun `a literal assigned to a VARCHAR(5) column still substitutes and reports NOT NULL`() { - // Proves format_type carried the column's typmod (length 5) rather than the substitution - // silently failing to build (which would fall back to the bare, unsubstituted `note` column - // — nullable in the schema — and report NULLABLE here instead). If the cast's type text were - // invalid SQL, or if the typmod were dropped in a way PostgreSQL rejected, the derived - // table's `PREPARE` would fail and the whole combined probe would fall back to the bare - // target, changing this answer. + // History: this used to prove format_type carried the column's typmod (length 5) into the + // derived table's cast, rather than the substitution silently failing to build and falling + // back to a bare, unsubstituted column. No derived table or cast exists today: "code"'s + // VARCHAR(5) type is the type PostgreSQL itself resolved when it parsed the real UPDATE + // statement, so there is nothing left that could fail to build or fall back. val query = analyzeWithSchema( "CREATE TABLE t (id INT PRIMARY KEY, code VARCHAR(5))", "UPDATE t SET code = 'ab' WHERE id = 1 RETURNING upper(code) AS n", @@ -6367,14 +6387,14 @@ class QueryAnalysisTest { @Test fun `an untyped range literal assigned to a DOMAIN over INT4RANGE reports nullable`() { - // A domain routes through CoerceToDomain, not a cast function, when the substituted - // expression is cast to the domain's own declared type (format_type reports the domain's - // own name, e.g. "r_domain", not its base type "int4range"). CoerceToDomain recurses - // unconditionally into its argument (see NodeTreeNullabilityAnalyzer.isNonNull), and - // PostgreSQL strips a domain down to its base type for function-overload resolution, so - // `lower()` here resolves the same anyrange overload it would against the real - // domain-typed column — reproducing the same wrong-overload bug this fix closes, but through - // CoerceToDomain instead of a cast function. + // History: a derived-table substitution used to re-resolve `lower()`'s overload against a + // separately-cast copy of "r", risking a different overload than the real statement's own + // parse chose. No re-resolution exists today: ColumnNullabilityAnalyzer.analyzeNodeTree's + // substitution (lines 539-557) reads the :targetList expression PostgreSQL itself built for + // the real statement — here a CoerceToDomain wrapping the assigned value — and + // NodeTreeNullabilityAnalyzer.isNonNull recurses through CoerceToDomain unconditionally, so + // this domain case cannot diverge from whatever overload the real RETURNING clause actually + // uses. val query = analyzeWithSchema( "CREATE DOMAIN r_domain AS INT4RANGE; CREATE TABLE t (id INT PRIMARY KEY, r r_domain)", "UPDATE t SET r = 'empty' WHERE id = 1 RETURNING lower(r) AS n", @@ -6385,15 +6405,17 @@ class QueryAnalysisTest { } /** - * [SetAssignmentAwareProbe]'s substitution ignored anything that rewrites the tuple between the - * `SET` clause and `RETURNING` — `RETURNING` always sees the final, post-trigger, post-rule - * tuple, never the raw `SET` expression, so a `BEFORE` row trigger, an `INSTEAD OF` trigger, a - * rewrite rule, or a foreign data wrapper's own write path - * can each substitute something else entirely for a value - * [PgCatalogLoader.buildUpdateSetAwareFromClause] would otherwise splice in as provably non-null. - * These tests exercise [PgCatalogLoader.targetRelationMayRewriteTupleBeforeReturning], the - * catalog-based bail that closes each of those gaps, plus the negative case (a statement-level or - * `AFTER` trigger) that proves the bail is targeted rather than a blanket "any trigger" check. + * [SetAssignmentAwareProbe]'s `:targetList`-to-`:returningList` substitution assumes `RETURNING` + * sees exactly the assigned value — but `RETURNING` always sees the final, post-trigger, + * post-rule tuple, never the raw `SET` expression, so a row-level `BEFORE` trigger, an + * `INSTEAD OF` trigger, a non-view rewrite rule, or a foreign data wrapper's own write path can + * each substitute something else entirely for a value that assumption would otherwise treat as + * provably non-null. These tests exercise + * [ColumnNullabilityAnalyzer.isSubstitutionSafeForRelation] (line 1260), which returns `false` — + * unsafe to trust — for exactly those cases on the target or any inheritance descendant, so + * [ColumnNullabilityAnalyzer.analyzeNodeTree] (line 556) leaves `:targetList` untrusted; plus + * the negative case (a statement-level or `AFTER` trigger) that proves the bail is targeted + * rather than a blanket "any trigger" check. */ @Nested inner class SetAssignmentAwareProbeCatalogBail { @@ -6517,13 +6539,15 @@ class QueryAnalysisTest { @Test @ResourceLock("postgres_fdw_loopback") fun `a foreign partition of a partitioned target reports nullable`() { - // Regression guard: targetRelationMayRewriteTupleBeforeReturning previously checked - // relkind only for the root relation ("p", a partitioned table — relkind 'p'), never for - // its descendants. A partition that is itself a FOREIGN TABLE (relkind 'f') was therefore - // invisible, and an FDW's own write path can produce any tuple it likes, independent of - // this statement's SET clause. PostgreSQL 18.4, via a postgres_fdw loopback with a BEFORE - // UPDATE row trigger on the remote table nulling "note": before the fix this reported NOT - // NULL; the actual RETURNING value is NULL. + // Regression guard: the predecessor to isSubstitutionSafeForRelation previously checked + // relkind only for the root relation ("p", a partitioned table — relkind 'p'), never for its + // descendants. A partition that is itself a FOREIGN TABLE (relkind 'f') was therefore + // invisible, and an FDW's own write path can produce any tuple it likes, independent of this + // statement's SET clause. PostgreSQL 18.4, via a postgres_fdw loopback with a BEFORE UPDATE + // row trigger on the remote table nulling "note": before the fix this reported NOT NULL; the + // actual RETURNING value is NULL. Today, isSubstitutionSafeForRelation's recursive + // pg_inherits CTE (lines 1263-1269) walks every descendant first, and its relkind check + // (line 1275) applies to each one, so a foreign partition can no longer be invisible. val schemaName = "test_${schemaCounter.incrementAndGet()}" DriverManager.getConnection(container.jdbcUrl, container.username, container.password).use { connection -> connection.createStatement().use { statement -> @@ -6649,39 +6673,20 @@ class QueryAnalysisTest { } /** - * Regression guard belonging to the same fix as [DmlReturning]'s three `merge_action()`/`OLD` - * abort guards, but exercising a data-modifying CTE body specifically, hence the `WITH` wrapper - * the other three tests intentionally omit. - * - * Coverage note on PgCatalogLoader's item-count-vs-real-column-count cross-check - * (`oldOrNewColumns.isNotEmpty() && oldOrNewAnalysis.itemCount != totalColumnCount`): this - * class's `an ALIASED star reaches the item-count cross-check directly` test below does reach - * this branch, reliably. `oldOrNewReturningColumns` (unlike `parseSelectItems`) never - * alias-strips an item before checking `isStarItem` against it — so any star carrying an - * alias, explicit (`tgt.* AS whatever`) or implicit (`tgt.* whatever`), is simply left - * unrecognized there: `isStarItem`'s own comment/whitespace/parenthesis normalization has no - * concept of an `AS` keyword or an implicit alias to look past, so the alias text survives - * normalization and the result never ends in `.*`. That unrecognized star's real expansion still - * shows up in the real column count, mismatching the assumed item count, which is exactly what - * this cross-check exists to catch — its outcome (forcing every column nullable) is the same - * safe over-approximation a recognized star produces via the `forcedColumns = null` path, just - * reached through the sibling branch instead. - * - * This is deliberately not "fixed" by alias-stripping inside `oldOrNewReturningColumns`: doing - * so would only change which branch produces the answer, never the answer itself (both branches - * force every column nullable), so there is no functional reason to add alias-awareness to this - * specific call site — and doing so would silently delete this branch's only current test - * coverage. This note makes no claim that this branch is otherwise unreachable in general — only - * that the test named above demonstrably reaches it today, via this specific alias-carrying - * shape. - * - * What the star-shape tests in this class (aside from the aliased-star one) still prove — see - * `an OLD reference without a star still forces only the referencing column, not the whole body` - * below — is that the per-column forcing mechanism (`forcedColumns` non-`null` and - * itemCount-matching) survives when no star is involved at all, distinguishing it from the - * whole-body `forceAllNullable` fallback every star-plus-`OLD`/`NEW` test in this class - * exercises (via one of the two possible routes: `forcedColumns == null`, or the item-count - * cross-check). + * Regression-guard suite for a family of historical bugs in a since-deleted text-based scanner + * that once had to recognize a RETURNING `*` item (in various spellings — parenthesized, + * whitespace-padded, comment-adjacent) and reconcile it against a separate OLD/NEW-reference + * scan, forcing every column nullable whenever the two interacted ambiguously. That whole + * scanner is gone: [PgNodeTreeParser.parseReturningList] reads PostgreSQL's own already-expanded + * target list — every `*`, however spelled, is expanded into individual `Var` nodes by + * PostgreSQL's own parser before Norm ever sees the RETURNING list — so each resulting `Var` (an + * ordinary column reference, or one tagged OLD/NEW by its own `:varreturningtype`) is evaluated + * independently by [NodeTreeNullabilityAnalyzer.isNonNull], with no cross-check between how many + * items were written and how many columns exist. Every test below is kept as a regression guard + * for the same end-to-end nullability outcome its historical bug once got wrong; none of the + * mechanisms the comments used to describe (a per-column forcing list, a dedicated + * OLD/NEW-returning-columns scan, an item-count cross-check, star-spelling recognition) exist + * today. */ @Nested inner class OldOrNewStarFailSafe { @@ -6689,21 +6694,16 @@ class QueryAnalysisTest { @Test fun `parenthesized star-plus-OLD in a CTE body forces every column, not just the wrong one`() { assumeTrue(pgVersion.substringBefore('.').toInt() >= 18, "RETURNING OLD requires PostgreSQL 18+") - // "(tgt.*)" is a parenthesized star — isStarItem's regex previously only recognized a bare - // "*"/"tbl.*", so this shape's real 2-column expansion (id, tval) was miscounted as a - // single RETURNING item, shifting "oldv" (the genuinely OLD-dependent column) off its real - // index and forcing the wrong column (tval) nullable instead of the real OLD-dependent - // column. isStarItem now recognizes the parenthesized form, so oldOrNewReturningColumns - // reports the mapping as known unreliable (forcedColumns = null) — the caller falls back to - // forcing every column nullable, same accepted over-approximation as the star-plus-OLD test - // elsewhere in this file, not an attempt at precisely isolating "oldv" alone. On real - // Postgres, with a fresh insert via ON CONFLICT — no prior row: id = 99, tval = 'x' (both - // genuinely NOT NULL — the just-inserted row's own columns), oldv = NULL (genuinely - // nullable) — but the safety net over-approximates all three to nullable here. + // "(tgt.*)" is a parenthesized star. PostgreSQL's own parser expands it into individual + // `Var`s for "id" and "tval" regardless of the parentheses — `PgNodeTreeParser.parseReturningList` + // reads that already-expanded list directly, with no star-recognition step of its own to get + // right or wrong. On real Postgres, with a fresh insert via ON CONFLICT — no prior row: id = + // 99, tval = 'x' (both genuinely NOT NULL — the just-inserted row's own columns), oldv = NULL + // (genuinely nullable). // prosqlbody reports "id" NOT NULL directly off its PRIMARY KEY catalog constraint — true // regardless of which INSERT/ON-CONFLICT branch actually ran. "tval" stays nullable: this // analyzer does not (yet) trace a CTE-nested INSERT's own :targetList/onConflict assignment - // the way it does for a top-level one (see PgCatalogLoader.analyzeNodeTree's targetListByResno + // the way it does for a top-level one (see ColumnNullabilityAnalyzer.analyzeNodeTree's targetListByResno // KDoc), so it falls back to tval's own (nullable) catalog constraint — safe, though not as // precise as the confirmed 'x' this comment already documents. "oldv" stays nullable via // the blanket OLD-forcing rule. @@ -6727,24 +6727,17 @@ class QueryAnalysisTest { fun `an ALIASED star reaches the item-count cross-check directly`() { assumeTrue(pgVersion.substringBefore('.').toInt() >= 18, "RETURNING OLD requires PostgreSQL 18+") // PostgreSQL 18.4: "RETURNING tgt.* AS whatever, OLD.id AS oldv" is valid syntax, returning - // 3 real columns for a 2-column "tgt" (tgt.id, tgt.tval, oldv). Unlike parseSelectItems, - // oldOrNewReturningColumns never alias-strips an item before checking isStarItem against - // it, so "tgt.* AS whatever" as a whole does not end in ".*" and is not recognized as a - // star here — hasRecognizedStarItem is false. oldOrNewColumnIndices still correctly - // identifies item 2 ("OLD.id AS oldv") as the OLD-referencing item, so - // oldOrNewReturningColumns returns forcedColumns = {2} (non-null) with itemCount = 2. Since - // the real column count is 3 (the star's own 2-column expansion was never counted), - // PgCatalogLoader's "oldOrNewColumns.isNotEmpty() && itemCount != totalColumnCount" check - // (2 != 3) fires and forces every column nullable — the item-count cross-check itself, not - // the "forcedColumns == null" branch the other star-plus-OLD tests in this class exercise. - // The outcome is the same safe over-approximation either way, which is exactly why this - // gap in oldOrNewReturningColumns's alias-awareness is not itself a bug: the cross-check - // makes the missed recognition harmless. On real Postgres, with a fresh insert via ON - // CONFLICT — no prior row: id = 99, tval = 'x' (both genuinely NOT NULL), oldv = NULL - // (genuinely nullable) — but the safety net over-approximates all three to nullable here. - // Same reasoning as the parenthesized-star test above: id NOT NULL via its PRIMARY KEY - // catalog constraint, tval nullable (CTE-nested INSERT assignment tracing not implemented — - // safe, not maximally precise), oldv nullable via the blanket OLD-forcing rule. + // 3 real columns for a 2-column "tgt" (tgt.id, tgt.tval, oldv). The alias on the star changes + // nothing about how PostgreSQL itself expands it: `tgt.*`'s two columns and `oldv` each + // arrive as their own `Var` in the already-expanded target list + // (`PgNodeTreeParser.parseReturningList`), independent of the alias text. On real Postgres, + // with a fresh insert via ON CONFLICT — no prior row: id = 99, tval = 'x' (both genuinely + // NOT NULL), oldv = NULL (genuinely nullable). "id" reports NOT NULL via its PRIMARY KEY + // catalog constraint; "tval" reports nullable (this analyzer does not yet trace a + // CTE-nested INSERT's own assignment the way it does for a top-level one — see + // `ColumnNullabilityAnalyzer.analyzeNodeTree`'s `targetListByResno` KDoc — so it falls back + // to tval's own nullable catalog constraint, safe though not maximally precise); "oldv" + // reports nullable via the blanket OLD-forcing rule. val query = analyzeWithSchema( "CREATE TABLE tgt (id INT PRIMARY KEY, tval TEXT)", """ @@ -6764,21 +6757,11 @@ class QueryAnalysisTest { @Test fun `star with whitespace around the dot is recognized by isStarItem and forces every column`() { assumeTrue(pgVersion.substringBefore('.').toInt() >= 18, "RETURNING OLD requires PostgreSQL 18+") - // isStarItem now collapses whitespace around a qualifying dot, so "tgt . *" is recognized — - // confirmed via SqlUtilsTest's "recognizes a star item with whitespace around the - // qualifying dot". That means oldOrNewReturningColumns itself now returns forcedColumns = - // null directly (a recognized star coincides with an OLD/NEW reference — see its KDoc), so - // PgCatalogLoader's forceNullableColumn short-circuits on "oldOrNewColumns == null" and - // never reaches the itemCount-vs-real-column-count cross-check below it — this test no - // longer exercises that cross-check at all (see the class KDoc). Kept as a regression guard - // for the observable end-to-end nullability outcome (which happens to be unchanged here, - // because "tgt" has two columns and the pre-fix itemCount already mismatched the real - // column count regardless of recognition — see `an OLD reference forces every column - // nullable when a star recognition change loses per-column precision` for a case where - // recognizing a star does change the observable outcome), not as cross-check coverage. On - // real Postgres, with a fresh insert via ON CONFLICT — no prior row, so OLD does not exist: - // id = 99, tval = 'y' (both genuinely NOT NULL), oldv = NULL (genuinely nullable). - // Same reasoning as the two star-plus-OLD tests above. + // Whitespace around the qualifying dot in "tgt . *" is ordinary SQL syntax PostgreSQL's + // own parser accepts and expands the same way as "tgt.*", regardless of spacing. On real + // Postgres, with a fresh insert via ON CONFLICT — no prior row, so OLD does not exist: id = + // 99, tval = 'y' (both genuinely NOT NULL), oldv = NULL (genuinely nullable). Same reasoning + // as the two star-plus-OLD tests above. val query = analyzeWithSchema( "CREATE TABLE tgt (id INT PRIMARY KEY, tval TEXT)", """ @@ -6798,24 +6781,14 @@ class QueryAnalysisTest { @Test fun `an OLD reference forces every column nullable when a star recognition change loses per-column precision`() { assumeTrue(pgVersion.substringBefore('.').toInt() >= 18, "RETURNING OLD requires PostgreSQL 18+") - // Pins a real, intentional nullability outcome change from teaching isStarItem to - // recognize "tgt . *" — not merely a different code path reaching the same answer, unlike - // the sibling tests above. "tgt" here has exactly one column, so "tgt.*"'s real expansion - // (1 column) plus "oldv" (1 column) happens to equal the assumed item count (2) — before - // isStarItem recognized "tgt . *", oldOrNewReturningColumns computed forcedColumns = {2} - // (only "oldv") with no itemCount mismatch to trigger the cross-check, so PgCatalogLoader's - // per-column forcing applied precisely: "id" (from tgt.*'s expansion) was governed by its - // real attnotnull (a PRIMARY KEY column — genuinely NOT NULL), and only "oldv" was forced. - // Now that "tgt . *" is recognized, oldOrNewReturningColumns returns forcedColumns = null - // directly, and PgCatalogLoader forces every column nullable — "id" included, even though - // it is genuinely NOT NULL. The direction is safe (over-nullable beats a fabricated NOT - // NULL elsewhere), which is why it is kept rather than reverted, but it is a real loss of - // precision for this specific shape, not merely a different route to an unchanged answer. - // On real Postgres, with a fresh insert via ON CONFLICT — no prior row, so OLD does not - // exist: id = 99 (genuinely NOT NULL), oldv = NULL (genuinely nullable). - // prosqlbody reports "id" NOT NULL directly off its PRIMARY KEY catalog constraint (the - // precision the pre-fix production code path happened to have here too), "oldv" nullable - // via the blanket OLD-forcing rule. + // Historical note: this once pinned a real, intentional precision regression from teaching + // a since-deleted text scanner to recognize "tgt . *" as a star. That scanner is gone: today + // each of "id" (from `tgt.*`'s expansion) and "oldv" is an independent `Var` in PostgreSQL's + // own already-expanded target list, resolved on its own terms — "id" via its PRIMARY KEY + // catalog constraint, "oldv" via the blanket OLD-forcing rule — so there is no cross-check + // between them left to lose precision. On real Postgres, with a fresh insert via ON + // CONFLICT — no prior row, so OLD does not exist: id = 99 (genuinely NOT NULL), oldv = NULL + // (genuinely nullable), matching what is asserted below. val query = analyzeWithSchema( "CREATE TABLE tgt (id INT PRIMARY KEY)", """ @@ -6834,13 +6807,12 @@ class QueryAnalysisTest { @Test fun `the same precision loss extends to one of the newly-normalized star spellings`() { assumeTrue(pgVersion.substringBefore('.').toInt() >= 18, "RETURNING OLD requires PostgreSQL 18+") - // Same outcome change as the test above, for one of the three spellings isStarItem was - // fixed to additionally normalize (a trailing comment sitting outside a wrapping - // parenthesis) — confirming the precision loss extends to those spellings too, exactly as - // expected: any input that flips from "unrecognized" to "recognized" on a one-column - // relation loses this same per-column precision. PostgreSQL 18.4: "(tgt .*) -- c" is valid - // syntax, id = 99 (genuinely NOT NULL), oldv = NULL (genuinely nullable). - // Same reasoning as the test above. + // Historical note, same shape as the test above for a different star spelling a + // since-deleted text scanner once needed to additionally normalize (a trailing comment + // sitting outside a wrapping parenthesis). That scanner is gone: PostgreSQL's own parser + // accepts and expands "(tgt.*) -- c" the same way regardless of the comment placement. + // PostgreSQL 18.4: id = 99 (genuinely NOT NULL), oldv = NULL (genuinely nullable), matching + // what is asserted below. val query = analyzeWithSchema( "CREATE TABLE tgt (id INT PRIMARY KEY)", "WITH u AS (\n" + @@ -6858,29 +6830,18 @@ class QueryAnalysisTest { @Test fun `an untracked bracket can no longer cancel out a star's split error and defeat the cross-check`() { assumeTrue(pgVersion.substringBefore('.').toInt() >= 18, "RETURNING OLD requires PostgreSQL 18+") - // splitAtTopLevel previously did not track "[...]", so "ARRAY[1, 2]"'s internal comma - // split it into two items — which, in this exact list, numerically canceled out the "tgt . *" - // star's own split error: 4 real columns (id, tval, oldv, arr), and a broken split - // ["tgt . *", "OLD.tval AS oldv", "ARRAY[1", "2] AS arr"] that also produced 4 items, - // defeating oldOrNewReturningColumns's real-column-count cross-check entirely and forcing - // the wrong column (the second half of "tgt.*"'s expansion, i.e. tval) instead of "oldv". - // isStarItem was later taught to recognize "tgt . *" (whitespace around the dot), so - // oldOrNewReturningColumns now returns forcedColumns = null directly for this list — a - // recognized star coincides with an OLD/NEW reference — and PgCatalogLoader's - // forceNullableColumn short-circuits on that null before ever comparing item count (3) - // against real column count (4). The item-count cross-check this test originally exercised - // is therefore not reached here anymore either; see the class KDoc. The observable outcome - // (all four forced nullable) is unchanged here regardless, because the pre-fix itemCount - // already mismatched the real column count on its own — see `an OLD reference forces every - // column nullable when a star recognition change loses per-column precision` for a case - // where recognizing a star does change the observable outcome. On real Postgres, with a + // Historical note: a since-deleted text-based item splitter once needed to track "[...]" + // so that "ARRAY[1, 2]"'s internal comma was not mistaken for an item separator. That + // splitter is gone: PostgreSQL's own parser resolves "ARRAY[1, 2]" as a single array-literal + // expression, and each RETURNING item — the star's own expansion, "oldv", and "arr" — + // arrives as its own already-parsed node with no text-level splitting involved at all. + // "id"/"tval"/"arr" are each reported NOT NULL or nullable from their own facts (PRIMARY KEY + // catalog constraint; nullable catalog constraint with CTE-nested-assignment tracing not yet + // implemented, see `ColumnNullabilityAnalyzer.analyzeNodeTree`'s `targetListByResno` KDoc; a + // genuine array-literal constructor, never itself NULL), and "oldv" is nullable via the + // blanket OLD-forcing rule — each independently of the others. On real Postgres, with a // fresh insert via ON CONFLICT — no prior row: id = 99, tval = 'x', arr = {1,2} (all - // genuinely NOT NULL), oldv = NULL (genuinely nullable) — the safety net still - // over-approximates all four to nullable, same accepted tradeoff as the other star-plus-OLD - // tests in this file, just reached via a different branch than before. - // prosqlbody: id NOT NULL (PRIMARY KEY catalog constraint), tval nullable (CTE-nested - // assignment tracing not implemented, safe not maximal), oldv nullable (blanket OLD-forcing), - // arr NOT NULL — ARRAY[1, 2] is a genuine array-literal constructor, never itself NULL. + // genuinely NOT NULL), oldv = NULL (genuinely nullable). val query = analyzeWithSchema( "CREATE TABLE tgt (id INT PRIMARY KEY, tval TEXT)", """ @@ -6901,24 +6862,13 @@ class QueryAnalysisTest { @Test fun `an untracked bracket alone, with no star at all, no longer corrupts the item count`() { assumeTrue(pgVersion.substringBefore('.').toInt() >= 18, "RETURNING OLD requires PostgreSQL 18+") - // This alone does not demonstrate the "[...]"-tracking fix: confirmed (via the established - // git-stash before/after technique) that this exact shape already passed at 94b5a2d — the - // untracked "[...]" corrupted the split into 3 items ("ARRAY[1", "2] AS arr", "OLD.tval AS - // oldv") against 2 real columns, and that mismatch (3 != 2) was already caught by the - // existing real-column-count cross-check, which forced every column nullable — - // coincidentally correct for "oldv" (genuinely nullable) even before this fix. What this - // does confirm is that the fix doesn't regress this shape: after tracking "[...]", the - // split is the correct 2 items, oldOrNewReturningColumns identifies "oldv" (not "arr") as - // the OLD-referencing item via precise per-column mapping rather than the coarser "force - // everything" fallback — a structural improvement even though it happens to produce the - // same observable nullability here. It does not prove "arr" keeps its true NOT NULL status - // either way: the stub path's own metadata probe reports a computed `ARRAY[]` expression's - // nullability as unknown/nullable regardless of forceNullableColumn, a separate, - // pre-existing imprecision of the metadata-probe stub itself, not of this fix. On real + // Historical note: same "[...]"-tracking bug as the test above, without a star at all. The + // since-deleted text splitter is gone; "ARRAY[1, 2]" and "OLD.tval AS oldv" each arrive as + // their own already-parsed node regardless of the bracket's internal comma. "arr" is a + // genuine array-literal constructor, never itself NULL, so it is reported NOT NULL directly; + // "oldv" is nullable via the blanket OLD-forcing rule, independently of "arr". On real // Postgres, with a fresh insert via ON CONFLICT: arr = {1,2} (genuinely NOT NULL), oldv = // NULL (genuinely nullable — no prior row). - // prosqlbody structurally recognizes ARRAY[1, 2] as a genuine array-literal constructor, - // never itself NULL — "oldv" stays nullable via the blanket OLD-forcing rule. val query = analyzeWithSchema( "CREATE TABLE tgt (id INT PRIMARY KEY, tval TEXT)", """ @@ -6937,20 +6887,13 @@ class QueryAnalysisTest { @Test fun `an OLD reference without a star still forces only the referencing column, not the whole body`() { assumeTrue(pgVersion.substringBefore('.').toInt() >= 18, "RETURNING OLD requires PostgreSQL 18+") - // Restores coverage lost when isStarItem was taught to recognize "tgt . *": every other - // test in this class now involves a star, so every one of them resolves via - // oldOrNewReturningColumns's forcedColumns = null (a recognized star coincides with an - // OLD/NEW reference) and PgCatalogLoader's whole-body forceAllNullable fallback — leaving - // nothing in this class asserting on the distinct per-column forcing mechanism - // (forcedColumns non-null, containing only the specific OLD/NEW-referencing item's index). - // With no star at all here, itemCount trivially matches the real column count, so - // oldOrNewMappingUnreliable is false and PgCatalogLoader's - // "oldOrNewColumns.orEmpty().contains(columnIndex)" line is what decides each column's - // fate. If that per-column check were ever replaced by forcing the whole body nullable - // whenever any OLD/NEW reference is present, "id" below would flip from NOT NULL to - // nullable and this assertion would fail. On real Postgres, with a fresh insert via ON - // CONFLICT — no prior row, so OLD does not exist: id = 99 (genuinely NOT NULL, the - // just-inserted row's own column), oldv = NULL (genuinely nullable). + // With no star at all, "id" and "oldv" are independent `Var`s in the RETURNING list — "id" + // is resolved by its own PRIMARY KEY catalog constraint, "oldv" by the blanket OLD-forcing + // rule — exactly as when a star is present elsewhere in the list (see the tests above): + // there has never been a route by which an OLD/NEW reference could force an unrelated column + // nullable today, star or no star. On real Postgres, with a fresh insert via ON CONFLICT — + // no prior row, so OLD does not exist: id = 99 (genuinely NOT NULL, the just-inserted row's + // own column), oldv = NULL (genuinely nullable). val query = analyzeWithSchema( "CREATE TABLE tgt (id INT PRIMARY KEY, tval TEXT)", """ @@ -6970,7 +6913,7 @@ class QueryAnalysisTest { /** * Set operations (UNION ALL, INTERSECT, EXCEPT) are conservatively treated as nullable even when * every branch selects NOT NULL columns. PostgreSQL represents set operations using subquery RTEs - * in the range table, and [PgCatalogLoader.buildSubqueryColumnNotNull] skips set-operation queries + * in the range table, and [ColumnNullabilityAnalyzer.buildSubqueryColumnNotNull] skips set-operation queries * to avoid incorrectly reporting the first branch's nullability as the whole result's nullability. * The target list VARs reference the first subquery RTE, whose varno has no entry in the base-table * range table, so [NodeTreeNullabilityAnalyzer] defaults to nullable. @@ -7269,7 +7212,7 @@ class QueryAnalysisTest { // A JSON_TABLE column resolves to a plain VAR against an RTE_TABLEFUNC entry (rtekind 4), // never to a JsonExpr the expression walk would hand to evaluateJsonExpr — the JSON_TABLE_OP // nodes PostgreSQL does emit live inside that RTE's :tablefunc, which this parser never - // descends into. parseRangeTable maps only rtekind 0, so the Var finds no entry, so nullable. + // descends into. baseRelations maps only rtekind 0, so the Var finds no entry, so nullable. assumeTrue(pgVersion.substringBefore('.').toInt() >= 17, "JSON_TABLE requires PostgreSQL 17+") val query = analyzeWithSchema( "CREATE TABLE t (id INT NOT NULL, doc JSONB NOT NULL)", @@ -7641,18 +7584,15 @@ class QueryAnalysisTest { } @Test - fun `ROW constructor correctly reports NOT NULL via the prosqlbody fallback`() { - // A bare, uncast ROW(...) produces an anonymous "record"-typed column, which PostgreSQL - // refuses to expose on a VIEW ("column result has pseudo-type record") — this SQL has no - // DML at all, but CREATE VIEW's failure routes it through queryColumnNullability's - // fallback regardless, since that fallback cannot distinguish "CREATE VIEW failed because - // of DML" from "CREATE VIEW failed for an unrelated reason". Before the prosqlbody cutover, - // that fallback was a text-based probe with no way to recognize a RowExpr's own shape, so it - // reported nullable rather than assert a proof it did not actually perform, even though a - // constructed ROW(...) value is, in fact, never itself NULL. The fallback is prosqlbody now: - // it is not subject to CREATE VIEW's pseudo-type rejection, and RowExpr is a node type its - // structural analysis already recognizes as never itself NULL. On real Postgres, `SELECT - // ROW(a, b) AS result FROM t` returns `result = (1,2)` for a matching row, never NULL. + fun `ROW constructor is correctly reported NOT NULL by prosqlbody, which CREATE VIEW itself would reject`() { + // A bare, uncast ROW(...) produces an anonymous "record"-typed column, which CREATE VIEW + // itself would reject ("column result has pseudo-type record") — but `queryColumnNullability` + // never attempts CREATE VIEW at all; every statement, this one included, is routed directly + // through `ColumnNullabilityAnalyzer.queryColumnNullabilityViaProsqlbody` + // (`PgCatalogLoader.queryColumnNullability`), which is not subject to that pseudo-type + // rejection. `RowExpr` is a node type `NodeTreeNullabilityAnalyzer` recognizes as never + // itself NULL. On real Postgres, `SELECT ROW(a, b) AS result FROM t` returns `result = + // (1,2)` for a matching row, never NULL. val query = analyzeWithSchema( "CREATE TABLE t (a INT NOT NULL, b INT NOT NULL)", "SELECT ROW(a, b) AS result FROM t", @@ -7661,17 +7601,14 @@ class QueryAnalysisTest { } @Test - fun `ROW constructor with a LEFT JOIN correctly reports NOT NULL via the prosqlbody fallback`() { - // This SQL has no DML at all, but CREATE VIEW rejects a bare ROW(...)'s anonymous "record" - // pseudo-type regardless, so it still reaches queryColumnNullability's fallback — prosqlbody - // now, in production. Before the cutover, that fallback's text-based null-extending-construct - // scan found the LEFT JOIN and forced every column nullable, including "t.a", which is - // declared NOT NULL and is never actually null-extended by this join (t is the LEFT side, - // not u) — an accepted, safe-direction imprecision at the time (a metadata-only answer that - // instead trusted the join structure would have been unsafe for the mirror case, `SELECT u.x - // FROM t LEFT JOIN u`, where `u.x` genuinely can be null-extended). prosqlbody reads the same - // :varnullingrels this whole cutover is built on, so it resolves both correctly without that - // tradeoff: on real Postgres, both columns are genuinely NOT NULL for a matching row. + fun `ROW constructor with a LEFT JOIN reports NOT NULL via prosqlbody, a route CREATE VIEW itself rejects`() { + // Same pseudo-type shape as the test above, plus a LEFT JOIN. `queryColumnNullability` + // routes this statement directly through prosqlbody with no CREATE VIEW attempt at all + // (`PgCatalogLoader.queryColumnNullability`), so "t.a" (the LEFT side, never null-extended + // by this join) and the ROW constructor over it are each resolved by their own + // `:varnullingrels`/`RowExpr` handling — genuinely independent per-column facts, not a + // whole-statement approximation. On real Postgres, both columns are genuinely NOT NULL for + // a matching row. val query = analyzeWithSchema( """ CREATE TABLE t (a INT NOT NULL, b INT NOT NULL); @@ -7916,8 +7853,7 @@ class QueryAnalysisTest { it.execute(chainDdl) } try { - val catalogLoader = PgCatalogLoader(connection) - val analyzer = ColumnNullabilityAnalyzer(catalogLoader) + val analyzer = ColumnNullabilityAnalyzer(connection, NullabilityCatalog(connection)) // Resolved directly from the DEEPEST view down, rather than through // loadViewColumnNullability's unordered schema-wide sweep, so this test is not at the // mercy of a resolution order that might happen to walk the chain shallow-first (and @@ -7960,11 +7896,12 @@ class QueryAnalysisTest { val boundaryRelid = regclassOid(connection, "boundary_1") val freshResult = ColumnNullabilityAnalyzer( - PgCatalogLoader(connection), + connection, + NullabilityCatalog(connection), ).resolveViewColumnNullability(deepestRelid) assertThat(freshResult).isEqualTo(listOf(true)) - val warmedAnalyzer = ColumnNullabilityAnalyzer(PgCatalogLoader(connection)) + val warmedAnalyzer = ColumnNullabilityAnalyzer(connection, NullabilityCatalog(connection)) warmedAnalyzer.resolveViewColumnNullability(boundaryRelid) val warmedResult = warmedAnalyzer.resolveViewColumnNullability(deepestRelid) assertThat(warmedResult).isEqualTo(freshResult) @@ -8064,8 +8001,7 @@ class QueryAnalysisTest { it.execute(chainDdl) } try { - val catalogLoader = PgCatalogLoader(connection) - val analyzer = ColumnNullabilityAnalyzer(catalogLoader) + val analyzer = ColumnNullabilityAnalyzer(connection, NullabilityCatalog(connection)) val deepestRelid = regclassOid(connection, "view_${depth - 1}") var result: List? = null @@ -8118,7 +8054,8 @@ class QueryAnalysisTest { // resolution order. val groundTruth = fixture.views.associateWith { view -> ColumnNullabilityAnalyzer( - PgCatalogLoader(connection), + connection, + NullabilityCatalog(connection), ).resolveViewColumnNullability(relidByView.getValue(view)) } @@ -8130,7 +8067,7 @@ class QueryAnalysisTest { for (warmView in fixture.warmTriggers) { for (checkView in fixture.views) { if (warmView == checkView) continue - val analyzer = ColumnNullabilityAnalyzer(PgCatalogLoader(connection)) + val analyzer = ColumnNullabilityAnalyzer(connection, NullabilityCatalog(connection)) analyzer.resolveViewColumnNullability(relidByView.getValue(warmView)) val afterWarming = analyzer.resolveViewColumnNullability(relidByView.getValue(checkView)) assertThat(afterWarming) @@ -8233,8 +8170,7 @@ class QueryAnalysisTest { ) } try { - val catalogLoader = PgCatalogLoader(connection) - val analyzer = ColumnNullabilityAnalyzer(catalogLoader) + val analyzer = ColumnNullabilityAnalyzer(connection, NullabilityCatalog(connection)) val topRelid = regclassOid(connection, "top") val aRelid = regclassOid(connection, "a") val midRelid = regclassOid(connection, "mid") @@ -8280,10 +8216,11 @@ class QueryAnalysisTest { val yRelid = regclassOid(connection, "y") val topvRelid = regclassOid(connection, "topv") - val groundTruthY = ColumnNullabilityAnalyzer(PgCatalogLoader(connection)).resolveViewColumnNullability(yRelid) + val groundTruthY = ColumnNullabilityAnalyzer(connection, NullabilityCatalog(connection)) + .resolveViewColumnNullability(yRelid) assertThat(groundTruthY).isEqualTo(listOf(false)) - val analyzer = ColumnNullabilityAnalyzer(PgCatalogLoader(connection)) + val analyzer = ColumnNullabilityAnalyzer(connection, NullabilityCatalog(connection)) val topvResult = analyzer.resolveViewColumnNullability(topvRelid) assertThat(topvResult).isEqualTo(listOf(true, true)) @@ -8328,8 +8265,7 @@ class QueryAnalysisTest { it.execute(ddl) } try { - val catalogLoader = PgCatalogLoader(connection) - val analyzer = ColumnNullabilityAnalyzer(catalogLoader) + val analyzer = ColumnNullabilityAnalyzer(connection, NullabilityCatalog(connection)) val topRelid = regclassOid(connection, "g_$diamondDepth") val elapsedMillis = System.nanoTime().let { startNanos -> @@ -8355,7 +8291,7 @@ class QueryAnalysisTest { // for the live sweep this backs), so this exercises the pure fallback logic directly with a // synthetic mismatch, rather than attempting to construct one via a real view. DriverManager.getConnection(container.jdbcUrl, container.username, container.password).use { connection -> - val analyzer = ColumnNullabilityAnalyzer(PgCatalogLoader(connection)) + val analyzer = ColumnNullabilityAnalyzer(connection, NullabilityCatalog(connection)) assertThat(analyzer.alignViewColumnNullability(listOf(false, true), expectedColumnCount = 3)) .isEqualTo(listOf(true, true, true)) @@ -8528,8 +8464,8 @@ class QueryAnalysisTest { it.execute("CREATE TABLE t (id INT PRIMARY KEY, name TEXT NOT NULL)") } try { - val catalogLoader = PgCatalogLoader(connection) - val result = ColumnNullabilityAnalyzer(catalogLoader).queryColumnNullabilityViaProsqlbody( + val analyzer = ColumnNullabilityAnalyzer(connection, NullabilityCatalog(connection)) + val result = analyzer.queryColumnNullabilityViaProsqlbody( """ MERGE INTO t USING (VALUES (1, 'new-name')) AS s(id, name) ON t.id = s.id WHEN NOT MATCHED THEN INSERT (id, name) VALUES (s.id, s.name) @@ -8558,8 +8494,8 @@ class QueryAnalysisTest { it.execute("CREATE TABLE t (id INT NOT NULL, name TEXT NOT NULL)") } try { - val catalogLoader = PgCatalogLoader(connection) - val result = ColumnNullabilityAnalyzer(catalogLoader).queryColumnNullabilityViaProsqlbody( + val analyzer = ColumnNullabilityAnalyzer(connection, NullabilityCatalog(connection)) + val result = analyzer.queryColumnNullabilityViaProsqlbody( """ SELECT id, name FROM t -- only active rows @@ -8585,8 +8521,8 @@ class QueryAnalysisTest { it.execute("CREATE TABLE t (id INT NOT NULL, name TEXT NOT NULL)") } try { - val catalogLoader = PgCatalogLoader(connection) - val result = ColumnNullabilityAnalyzer(catalogLoader).queryColumnNullabilityViaProsqlbody( + val analyzer = ColumnNullabilityAnalyzer(connection, NullabilityCatalog(connection)) + val result = analyzer.queryColumnNullabilityViaProsqlbody( "SELECT id, name FROM t /* only active rows */", ) assertThat(result?.map { it.nullable }).isEqualTo(listOf(false, false)) @@ -8611,8 +8547,8 @@ class QueryAnalysisTest { it.execute("CREATE TABLE t (id INT NOT NULL, name TEXT NOT NULL)") } try { - val catalogLoader = PgCatalogLoader(connection) - val result = ColumnNullabilityAnalyzer(catalogLoader).queryColumnNullabilityViaProsqlbody( + val analyzer = ColumnNullabilityAnalyzer(connection, NullabilityCatalog(connection)) + val result = analyzer.queryColumnNullabilityViaProsqlbody( """ SELECT id, name FROM t /* unterminated @@ -8637,8 +8573,8 @@ class QueryAnalysisTest { it.execute("CREATE TABLE t (id INT NOT NULL, name TEXT NOT NULL)") } try { - val catalogLoader = PgCatalogLoader(connection) - val result = ColumnNullabilityAnalyzer(catalogLoader).queryColumnNullabilityViaProsqlbody( + val analyzer = ColumnNullabilityAnalyzer(connection, NullabilityCatalog(connection)) + val result = analyzer.queryColumnNullabilityViaProsqlbody( "SELECT id, name FROM t WHERE name = \$\$unterminated", ) assertThat(result).isNull() @@ -8660,8 +8596,8 @@ class QueryAnalysisTest { // happen to be true and the always-non-null set happens to be non-empty — the bare // `isNotEmpty()` this replaces would stay green even if strictness were loaded backwards. DriverManager.getConnection(container.jdbcUrl, container.username, container.password).use { connection -> - val catalogLoader = PgCatalogLoader(connection) - val strictness = catalogLoader.functionStrictnessByOid + val catalog = NullabilityCatalog(connection) + val strictness = catalog.functionStrictnessByOid val oidsByPronameAndArgtypes = connection.createStatement().use { stmt -> stmt.executeQuery( """ @@ -8703,8 +8639,8 @@ class QueryAnalysisTest { // claims — then spot-checks that sum/avg/max/min, which the map correctly excludes, really // do return null over the same empty input. DriverManager.getConnection(container.jdbcUrl, container.username, container.password).use { connection -> - val catalogLoader = PgCatalogLoader(connection) - val nonNullInitialOids = catalogLoader.aggregateHasNonNullInitialValue.filterValues { it }.keys + val catalog = NullabilityCatalog(connection) + val nonNullInitialOids = catalog.aggregateHasNonNullInitialValue.filterValues { it }.keys assertThat(nonNullInitialOids.isNotEmpty()).isTrue() val signaturesByOid = connection.createStatement().use { stmt -> @@ -8753,7 +8689,7 @@ class QueryAnalysisTest { @Test fun `immutableFunctionOids includes an immutable example and excludes stable, volatile, and set-returning ones`() { // Deliberately does not compare immutableFunctionOids against a live re-query of - // `provolatile = 'i' AND NOT proretset AND prokind IN ('f', 'w')` — PgCatalogLoader.loadImmutableFunctionOids + // `provolatile = 'i' AND NOT proretset AND prokind IN ('f', 'w')` — NullabilityCatalog.loadImmutableFunctionOids // runs exactly that predicate (see PgCatalogLoader.kt), so an `isEqualTo` against the same // predicate here could only ever detect broken plumbing (a query that fails to run at all), // never a wrong predicate: a mutation to the production SQL would move both sides of the @@ -8764,8 +8700,8 @@ class QueryAnalysisTest { // (`provolatile = 'i'`) but set-returning (`proretset = true`), so it must be excluded by the // `NOT proretset` conjunct specifically, not by volatility. DriverManager.getConnection(container.jdbcUrl, container.username, container.password).use { connection -> - val catalogLoader = PgCatalogLoader(connection) - val immutableOids = catalogLoader.immutableFunctionOids + val catalog = NullabilityCatalog(connection) + val immutableOids = catalog.immutableFunctionOids assertThat(immutableOids.isNotEmpty()).isTrue() val exampleOids = connection.createStatement().use { stmt -> @@ -8809,15 +8745,15 @@ class QueryAnalysisTest { @Test fun `neverNullForNonNullInputOids and lagLeadWithDefaultOids contain no VARIADIC pg_proc rows`() { - // PgCatalogLoader.neverNullForNonNullInputOids's KDoc claims no function on that list is + // NullabilityCatalog.neverNullForNonNullInputOids's KDoc claims no function on that list is // VARIADIC, checked on PostgreSQL 16-18 — unlike alwaysNonNullFunctionOids and // nonNullIffFirstArgumentNonNullFunctionOids, whose sole entries (concat/concat_ws) are // deliberately, documented VARIADIC ("any") and are excluded from this check for exactly // that reason (see PgNodeExpression.FuncExpr.isVariadic's KDoc and the two properties' own // KDoc for why the VARIADIC calling form is handled separately rather than trusted here). DriverManager.getConnection(container.jdbcUrl, container.username, container.password).use { connection -> - val catalogLoader = PgCatalogLoader(connection) - val oidsToCheck = catalogLoader.neverNullForNonNullInputOids + catalogLoader.lagLeadWithDefaultOids + val catalog = NullabilityCatalog(connection) + val oidsToCheck = catalog.neverNullForNonNullInputOids + catalog.lagLeadWithDefaultOids assertThat(oidsToCheck.isNotEmpty()).isTrue() val variadicOids = connection.createStatement().use { stmt -> stmt.executeQuery("SELECT oid::integer FROM pg_catalog.pg_proc WHERE provariadic != 0").use { rs -> @@ -9004,8 +8940,8 @@ class QueryAnalysisTest { @Test fun `neverNullForNonNullInput OIDs are loaded and exclude JSON path operators`() { DriverManager.getConnection(container.jdbcUrl, container.username, container.password).use { connection -> - val catalogLoader = PgCatalogLoader(connection) - val safeListed = catalogLoader.neverNullForNonNullInputOids + val catalog = NullabilityCatalog(connection) + val safeListed = catalog.neverNullForNonNullInputOids assertThat(safeListed.isNotEmpty()).isTrue() // "->" and "->>" (JSON path extraction) are STRICT but not TOTAL — they return null on a @@ -9205,7 +9141,7 @@ class QueryAnalysisTest { // on 16/17. Asserting the live server's actual pairs equal that fixed snapshot directly (as // a prior version of this test did) therefore fails on 16/17 even though nothing about those // six pairs is wrong there — they just do not resolve on this connected server, exactly the - // way PgCatalogLoader.loadNeverNullForNonNullInputOids's live catalog lookup finds no row + // way NullabilityCatalog.loadNeverNullForNonNullInputOids's live catalog lookup finds no row // for them either. // // The fix keeps the assertion an exact equality (never weakened to a subset check) by @@ -9310,11 +9246,11 @@ class QueryAnalysisTest { // consistency with every sibling test in this class (each PgCatalogLoader-loaded property // gets its own small, cheap, exact-by-name OID check here) — but upgraded from a bare // `isNotEmpty()` to genuine exact membership, resolved by name from pg_catalog independently - // of PgCatalogLoader.loadLagLeadWithDefaultOids's own `pronargs = 3` predicate, so a mutation + // of NullabilityCatalog.loadLagLeadWithDefaultOids's own `pronargs = 3` predicate, so a mutation // that widens or narrows that predicate (e.g. to the 2-argument overloads) is caught by the // resulting set inequality here too, not just in the heavier live sweep. DriverManager.getConnection(container.jdbcUrl, container.username, container.password).use { connection -> - val catalogLoader = PgCatalogLoader(connection) + val catalog = NullabilityCatalog(connection) val expectedOids = connection.createStatement().use { stmt -> stmt.executeQuery( """ @@ -9326,7 +9262,7 @@ class QueryAnalysisTest { ).use { rs -> buildSet { while (rs.next()) add(rs.getInt("oid")) } } } assertThat(expectedOids.isNotEmpty()).isTrue() - assertThat(catalogLoader.lagLeadWithDefaultOids).isEqualTo(expectedOids) + assertThat(catalog.lagLeadWithDefaultOids).isEqualTo(expectedOids) } } @@ -9341,7 +9277,7 @@ class QueryAnalysisTest { // concurrently creates and drops schema-scoped `concat`/`concat_ws` functions that an // un-namespaced query could intermittently pick up. DriverManager.getConnection(container.jdbcUrl, container.username, container.password).use { connection -> - val catalogLoader = PgCatalogLoader(connection) + val catalog = NullabilityCatalog(connection) val concatOids = connection.createStatement().use { stmt -> stmt.executeQuery( """ @@ -9353,7 +9289,7 @@ class QueryAnalysisTest { ).use { rs -> buildSet { while (rs.next()) add(rs.getInt("oid")) } } } assertThat(concatOids.isNotEmpty()).isTrue() - assertThat(catalogLoader.alwaysNonNullFunctionOids).isEqualTo(concatOids) + assertThat(catalog.alwaysNonNullFunctionOids).isEqualTo(concatOids) val concatWsOids = connection.createStatement().use { stmt -> stmt.executeQuery( @@ -9365,7 +9301,7 @@ class QueryAnalysisTest { """.trimIndent(), ).use { rs -> buildSet { while (rs.next()) add(rs.getInt("oid")) } } } - assertThat(catalogLoader.alwaysNonNullFunctionOids.intersect(concatWsOids)).isEqualTo(emptySet()) + assertThat(catalog.alwaysNonNullFunctionOids.intersect(concatWsOids)).isEqualTo(emptySet()) } } @@ -9374,7 +9310,7 @@ class QueryAnalysisTest { // `isNotEmpty()` alone is the exact `size >= 2` shape that let the original concat_ws bug // ship: it stays green under a mutation that puts `concat` (or any other function) on this // list alongside or instead of `concat_ws`, since the set is still non-empty either way. The - // OIDs resolved here are independent of PgCatalogLoader.loadNonNullIffFirstArgumentNonNullFunctionOids's + // OIDs resolved here are independent of NullabilityCatalog.loadNonNullIffFirstArgumentNonNullFunctionOids's // own SQL — matched by name alone, not by re-asserting its `NOT proisstrict` predicate — so a // mutation that widens or narrows the production predicate to a different name (or an // additional one) is caught by the resulting set inequality, not absorbed by both sides @@ -9385,7 +9321,7 @@ class QueryAnalysisTest { // drops a schema-scoped `concat_ws` — an un-namespaced query here would intermittently pick up // that shadow OID too and fail this exact-equality assertion on a false positive. DriverManager.getConnection(container.jdbcUrl, container.username, container.password).use { connection -> - val catalogLoader = PgCatalogLoader(connection) + val catalog = NullabilityCatalog(connection) val concatWsOids = connection.createStatement().use { stmt -> stmt.executeQuery( """ @@ -9397,7 +9333,7 @@ class QueryAnalysisTest { ).use { rs -> buildSet { while (rs.next()) add(rs.getInt("oid")) } } } assertThat(concatWsOids.isNotEmpty()).isTrue() - assertThat(catalogLoader.nonNullIffFirstArgumentNonNullFunctionOids).isEqualTo(concatWsOids) + assertThat(catalog.nonNullIffFirstArgumentNonNullFunctionOids).isEqualTo(concatWsOids) } } @@ -9435,9 +9371,9 @@ class QueryAnalysisTest { rs.getInt("oid") } } - val catalogLoader = PgCatalogLoader(connection) - assertThat(catalogLoader.alwaysNonNullFunctionOids.contains(shadowConcatOid)).isFalse() - assertThat(catalogLoader.nonNullIffFirstArgumentNonNullFunctionOids.contains(shadowConcatWsOid)).isFalse() + val catalog = NullabilityCatalog(connection) + assertThat(catalog.alwaysNonNullFunctionOids.contains(shadowConcatOid)).isFalse() + assertThat(catalog.nonNullIffFirstArgumentNonNullFunctionOids.contains(shadowConcatWsOid)).isFalse() } finally { connection.createStatement().use { it.execute("DROP SCHEMA $schemaName CASCADE") } } @@ -9451,6 +9387,27 @@ class QueryAnalysisTest { catalogLoader.checkPostgresVersion() } } + + @Test + fun `two NullabilityCatalog instances on the same connection agree on functionStrictnessByOid`() { + // Pins that functionStrictnessByOid is a pure catalog read with no hidden instance state + // involved: two independently constructed catalogs against the identical connection must load + // the identical map. + // + // Compared over built-in rows only. Tests share one container and run in parallel, so a + // sibling test can CREATE and DROP a function between the two loads, and that row's OID then + // appears in one map and not the other -- an observed failure, not a hypothetical + // (oid 20259). Every OID below FIRST_NORMAL_OBJECT_ID was allocated when the cluster was + // initialised and no test can add or drop one, so restricting to that range removes the race + // without weakening what the assertion proves: a map built twice from the same rows must + // agree, and any per-instance state would show up here just as plainly. + DriverManager.getConnection(container.jdbcUrl, container.username, container.password).use { connection -> + val first = NullabilityCatalog(connection) + val second = NullabilityCatalog(connection) + assertThat(first.functionStrictnessByOid.filterKeys { it < FIRST_NORMAL_OBJECT_ID }) + .isEqualTo(second.functionStrictnessByOid.filterKeys { it < FIRST_NORMAL_OBJECT_ID }) + } + } } @Nested @@ -9458,11 +9415,11 @@ class QueryAnalysisTest { private val schema = "CREATE TABLE t (id BIGINT PRIMARY KEY, a TEXT, b TEXT, flag BOOLEAN)" - // A DML-to-SELECT conversion (see PgCatalogLoader.transformForViewCreation / - // SqlUtils.convertDmlToSelect) drops the SET clause but keeps the original WHERE predicate. // A qual that looks like it proves a RETURNING column non-null may really be testing a value - // the statement is about to overwrite, so qual narrowing must be suppressed entirely whenever - // the analyzed SQL passed through that conversion. + // the statement's own SET clause (or, for MERGE, an update/insert action) is about to + // overwrite, so qual narrowing must be suppressed entirely for a data-modifying query block. + // ColumnNullabilityAnalyzer.buildQueryBlockScope (line 1102) computes qualProvenVars empty + // whenever resultRelationVarno != 0 — see QueryBlockScope's own KDoc (lines 99-102). private val dmlSchema = """ CREATE TABLE t (id INT NOT NULL, a TEXT); CREATE TABLE u (id INT NOT NULL, val TEXT) @@ -9842,16 +9799,16 @@ class QueryAnalysisTest { } @Test - fun `parseRangeTable returns empty for malformed input`() { + fun `baseRelations returns empty for malformed input`() { val parser = PgNodeTreeParser() - val result = parser.parseRangeTable("malformed") + val result = parser.parseRangeTableEntries("malformed").baseRelations() assertThat(result).hasSize(0) } @Test - fun `parseGroupRteMap returns empty for malformed input`() { + fun `groupRteMap returns empty for malformed input`() { val parser = PgNodeTreeParser() - val result = parser.parseGroupRteMap("malformed") + val result = parser.parseRangeTableEntries("malformed").groupRteMap(parser) assertThat(result).hasSize(0) } @@ -9979,7 +9936,7 @@ class QueryAnalysisTest { } /** - * Ground truth for [PgCatalogLoader.aggregateHasNonNullInitialValue]: calls the aggregate named + * Ground truth for [NullabilityCatalog.aggregateHasNonNullInitialValue]: calls the aggregate named * [aggregateName], with arguments typed [argumentTypeNames], over a genuinely empty input (`... * WHERE false`, not merely an aggregate over no matching group), and reports whether the result * is `null`. `count` with zero declared arguments is `count(*)` — the only ordinary-call-syntax diff --git a/generator/src/test/kotlin/norm/generator/QueryFileParserTest.kt b/generator/src/test/kotlin/norm/generator/QueryFileParserTest.kt index 05082410..fa386e4a 100644 --- a/generator/src/test/kotlin/norm/generator/QueryFileParserTest.kt +++ b/generator/src/test/kotlin/norm/generator/QueryFileParserTest.kt @@ -389,4 +389,98 @@ class QueryFileParserTest { assertThat(result[0].sql).isEqualTo("SELECT * FROM users WHERE name LIKE ':notaparam' AND id = ?") assertThat(result[0].namedParameters).isEqualTo(mapOf(1 to "id")) } + + @Test + fun `named parameter inside a block comment is not converted`() { + val content = """ + -- name: findByPattern :many + SELECT * FROM users /* :notaparam */ WHERE id = :id; + """.trimIndent() + + val result = QueryFileParser.parse(content) + + assertThat(result[0].sql).isEqualTo("SELECT * FROM users /* :notaparam */ WHERE id = ?") + assertThat(result[0].namedParameters).isEqualTo(mapOf(1 to "id")) + } + + @Test + fun `named parameter inside a quoted identifier is not converted`() { + val content = """ + -- name: findByPattern :many + SELECT "a:b" FROM users WHERE id = :id; + """.trimIndent() + + val result = QueryFileParser.parse(content) + + assertThat(result[0].sql).isEqualTo("""SELECT "a:b" FROM users WHERE id = ?""") + assertThat(result[0].namedParameters).isEqualTo(mapOf(1 to "id")) + } + + @Test + fun `named parameter inside a dollar-quoted string is not converted`() { + val content = """ + -- name: findByPattern :many + SELECT ${'$'}${'$'}:notaparam${'$'}${'$'} FROM users WHERE id = :id; + """.trimIndent() + + val result = QueryFileParser.parse(content) + + assertThat(result[0].sql).isEqualTo("SELECT \$\$:notaparam\$\$ FROM users WHERE id = ?") + assertThat(result[0].namedParameters).isEqualTo(mapOf(1 to "id")) + } + + @Test + fun `question mark inside a string literal does not trip the mixed-style guard`() { + val content = """ + -- name: findByPattern :many + SELECT * FROM users WHERE note = 'really?' AND id = :id; + """.trimIndent() + + val result = QueryFileParser.parse(content) + + assertThat(result[0].sql).isEqualTo("SELECT * FROM users WHERE note = 'really?' AND id = ?") + assertThat(result[0].namedParameters).isEqualTo(mapOf(1 to "id")) + } + + @Test + fun `genuinely mixed named and positional parameters still throw`() { + val content = """ + -- name: findByPattern :many + SELECT * FROM users WHERE note = ? AND id = :id; + """.trimIndent() + + assertFailure { QueryFileParser.parse(content) } + .messageContains("mix") + } + + @Test + fun `named parameter after the inner close of a nested block comment is not converted`() { + val content = """ + -- name: findByPattern :many + SELECT * FROM users /* a /* b */ :notaparam */ WHERE id = :id; + """.trimIndent() + + val result = QueryFileParser.parse(content) + + assertThat(result[0].sql).isEqualTo("SELECT * FROM users /* a /* b */ :notaparam */ WHERE id = ?") + assertThat(result[0].namedParameters).isEqualTo(mapOf(1 to "id")) + } + + @Test + fun `named-parameter-shaped text after a backslash-escaped quote in an E-string is not converted`() { + // The old findClosingQuote ended the literal at the backslash-escaped quote (it had no E-string + // awareness), so it would have read '\'' as the terminator and converted the ":x" that follows + // as a real named parameter. skipLexicalToken understands E'...' backslash escapes, so the + // whole literal -- ":x" included -- is skipped as one token. Accepted behavior change, matching + // PostgreSQL's own E'' semantics. + val content = """ + -- name: findByPattern :many + SELECT * FROM users WHERE note = E'it\'s :x' AND id = :id; + """.trimIndent() + + val result = QueryFileParser.parse(content) + + assertThat(result[0].sql).isEqualTo("""SELECT * FROM users WHERE note = E'it\'s :x' AND id = ?""") + assertThat(result[0].namedParameters).isEqualTo(mapOf(1 to "id")) + } } diff --git a/generator/src/test/kotlin/norm/generator/SafeListSweepTest.kt b/generator/src/test/kotlin/norm/generator/SafeListSweepTest.kt index 8d9b8b2c..ca928d42 100644 --- a/generator/src/test/kotlin/norm/generator/SafeListSweepTest.kt +++ b/generator/src/test/kotlin/norm/generator/SafeListSweepTest.kt @@ -169,7 +169,7 @@ class SafeListSweepTest { /** * pgcrypto's `digest`/`hmac` are keyed through `pg_depend` rather than appearing on any of the - * three static safe lists (see [PgCatalogLoader.loadNeverNullForNonNullInputOids]'s pgcrypto + * three static safe lists (see [NullabilityCatalog.loadNeverNullForNonNullInputOids]'s pgcrypto * carve-out), so none of the three tests above exercises them. This test runs the same * brute-force sweep over all four documented-total overloads (`digest(text, text)`, * `digest(bytea, text)`, `hmac(text, text, text)`, `hmac(bytea, bytea, text)`). @@ -444,7 +444,7 @@ class SafeListSweepTest { } /** - * Brute-force verification that every entry in [PgCatalogLoader.alwaysNonNullFunctionOids] + * Brute-force verification that every entry in [NullabilityCatalog.alwaysNonNullFunctionOids] * really is non-null for any combination of argument values, including when every argument is * `NULL` — the exact claim [concat_ws] shipping on this list would have violated (`concat_ws` * returns `null` when its separator is `NULL`, even though every other argument is non-null). @@ -453,7 +453,7 @@ class SafeListSweepTest { * time (with every other position drawn from [EDGE_VALUE_CORPUS]), plus the all-`NULL` * combination. * - * OIDs are read from [PgCatalogLoader.alwaysNonNullFunctionOids] itself — computed live, the same + * OIDs are read from [NullabilityCatalog.alwaysNonNullFunctionOids] itself — computed live, the same * way production does — rather than a hardcoded OID, and resolved back to a `pg_proc.proname` via * [resolveProcName] so this test automatically covers whatever the production list actually * contains today. [NULL_ARGUMENT_SWEEP_SIGNATURES_BY_NAME] supplies the concrete arity/types to @@ -463,8 +463,8 @@ class SafeListSweepTest { */ @Test fun `every alwaysNonNullFunctionOids-listed function is non-null for every NULL-argument combination`() { - val catalogLoader = PgCatalogLoader(connection) - val oids = catalogLoader.alwaysNonNullFunctionOids + val catalog = NullabilityCatalog(connection) + val oids = catalog.alwaysNonNullFunctionOids assertThat(oids.isNotEmpty()).isTrue() val failures = mutableListOf() var caseCount = 0 @@ -494,7 +494,7 @@ class SafeListSweepTest { /** * Brute-force verification of both directions of - * [PgCatalogLoader.nonNullIffFirstArgumentNonNullFunctionOids]'s claim: (i) a non-null first + * [NullabilityCatalog.nonNullIffFirstArgumentNonNullFunctionOids]'s claim: (i) a non-null first * argument with `NULL`(s) anywhere else never produces a `null` result, and (ii) a `NULL` first * argument always produces a `null` result, regardless of the other arguments. Property (ii) is * what distinguishes this list from [alwaysNonNullFunctionOids] — an entry here is non-null only @@ -506,8 +506,8 @@ class SafeListSweepTest { */ @Test fun `every nonNullIffFirstArgumentNonNullFunctionOids-listed function depends only on its first argument`() { - val catalogLoader = PgCatalogLoader(connection) - val oids = catalogLoader.nonNullIffFirstArgumentNonNullFunctionOids + val catalog = NullabilityCatalog(connection) + val oids = catalog.nonNullIffFirstArgumentNonNullFunctionOids assertThat(oids.isNotEmpty()).isTrue() val failures = mutableListOf() var caseCount = 0 @@ -557,7 +557,7 @@ class SafeListSweepTest { } /** - * Verification of [PgCatalogLoader.lagLeadWithDefaultOids]'s claim: the 3-argument `lag`/`lead` + * Verification of [NullabilityCatalog.lagLeadWithDefaultOids]'s claim: the 3-argument `lag`/`lead` * overloads are non-null when their value and default expressions are non-null, even at a window * boundary where the 1- and 2-argument forms would return `null` (no such row exists to fetch). * Runs both `lag` and `lead` over a small non-null, ordered dataset with a non-null literal @@ -569,8 +569,8 @@ class SafeListSweepTest { */ @Test fun `lagLeadWithDefaultOids-listed 3-argument lag and lead fill window boundaries from a non-null default`() { - val catalogLoader = PgCatalogLoader(connection) - val threeArgumentOids = catalogLoader.lagLeadWithDefaultOids + val catalog = NullabilityCatalog(connection) + val threeArgumentOids = catalog.lagLeadWithDefaultOids assertThat(threeArgumentOids.isNotEmpty()).isTrue() val actualThreeArgumentOids = connection.createStatement().use { stmt -> @@ -723,7 +723,7 @@ class SafeListSweepTest { * pgcrypto's `digest`/`hmac` overloads, brute-force-swept for total-ness the same way as the * three static [PgCatalogLoader] safe lists, but not sourced from any of them: they are an * extension carve-out keyed through `pg_depend`, not a name/argument-type entry on a list (see - * [PgCatalogLoader.loadNeverNullForNonNullInputOids]). Defined here, in the test, rather than + * [NullabilityCatalog.loadNeverNullForNonNullInputOids]). Defined here, in the test, rather than * in production code, since nothing else needs a [SafeFunctionSignature] for them. */ private val PGCRYPTO_FUNCTION_SIGNATURES = listOf( @@ -798,11 +798,11 @@ class SafeListSweepTest { private val SELF_CAST_SOURCE_TYPMOD_SUFFIX: Map = mapOf("bit" to "(3)") /** - * Concrete arities/types to sweep for [PgCatalogLoader.alwaysNonNullFunctionOids]'s and - * [PgCatalogLoader.nonNullIffFirstArgumentNonNullFunctionOids]'s NULL-argument properties, + * Concrete arities/types to sweep for [NullabilityCatalog.alwaysNonNullFunctionOids]'s and + * [NullabilityCatalog.nonNullIffFirstArgumentNonNullFunctionOids]'s NULL-argument properties, * keyed by `pg_proc.proname`. Both properties are keyed by name alone in production (see - * [PgCatalogLoader.loadAlwaysNonNullFunctions]/ - * [PgCatalogLoader.loadNonNullIffFirstArgumentNonNullFunctionOids]'s `proname = '...'` + * [NullabilityCatalog.loadAlwaysNonNullFunctions]/ + * [NullabilityCatalog.loadNonNullIffFirstArgumentNonNullFunctionOids]'s `proname = '...'` * predicates), because `concat`/`concat_ws`'s single `pg_catalog` row for each is declared * `VARIADIC "any"`/`VARIADIC "any"` — the declared argument type is the pseudo-type `any` * itself, with no literal form of its own, unlike `anyarray`/`anyrange`/`anyelement`, which @@ -810,7 +810,7 @@ class SafeListSweepTest { * arity/types actually exercised at the call site instead. * * `concat_ws` is registered here even though production correctly never lists it under - * [PgCatalogLoader.alwaysNonNullFunctionOids] today — if that regressed (this is exactly the + * [NullabilityCatalog.alwaysNonNullFunctionOids] today — if that regressed (this is exactly the * shipped bug this whole file's KDoc describes), the always-non-null sweep must actually * exercise `concat_ws`'s real NULL-argument behavior and fail on the genuine semantic violation * (`concat_ws(NULL, 'x', 'y')` returns `null`), not merely fail on a missing corpus @@ -955,7 +955,7 @@ class SafeListSweepTest { /** * `true` when [signature] resolves to a real `pg_proc` row on this connected server, checked - * by the same lookup [PgCatalogLoader.loadNeverNullForNonNullInputOids] performs in + * by the same lookup [NullabilityCatalog.loadNeverNullForNonNullInputOids] performs in * production (name plus the exact ordered list of declared argument `pg_type.typname` * values, restricted to `pronamespace = 'pg_catalog'`) — just run per-signature here instead * of batched. This is the independent check [neverResolvedOnThisServer]'s KDoc says every @@ -995,7 +995,7 @@ class SafeListSweepTest { * `true` when [signature] resolves to a real, `castfunc`-backed `pg_cast` row on this * connected server — the cast analogue of [functionResolvesOnThisServer], mirroring the same * `pg_cast`/`pg_type`/`pg_proc`/`pg_namespace` lookup - * [PgCatalogLoader.loadNeverNullForNonNullInputOids] performs in production. + * [NullabilityCatalog.loadNeverNullForNonNullInputOids] performs in production. */ private fun castResolvesOnThisServer(signature: SafeCastSignature): Boolean { val sql = """ @@ -1026,7 +1026,7 @@ class SafeListSweepTest { * `true` when [signature] resolves to a real, `oprcode`-backed `pg_operator` row on this * connected server — the operator analogue of [functionResolvesOnThisServer], mirroring the * same `pg_operator`/`pg_type`/`pg_namespace` lookup - * [PgCatalogLoader.loadNeverNullForNonNullInputOids] performs in production. A `null` + * [NullabilityCatalog.loadNeverNullForNonNullInputOids] performs in production. A `null` * [SafeOperatorSignature.leftTypeName]/[SafeOperatorSignature.rightTypeName] means the * operator has no operand on that side (a prefix/postfix operator), matched with `IS NOT * DISTINCT FROM` the same way production's bulk query does. @@ -1142,7 +1142,7 @@ private class CoverageTracker(private val minimumPositiveCases: Int = 1) { * cross-checks the resulting skip set against an INDEPENDENT catalog-only resolution check * (`functionResolvesOnThisServer`/`castResolvesOnThisServer`/`operatorResolvesOnThisServer`, * the same `pg_proc`/`pg_cast`/`pg_operator` lookup - * [PgCatalogLoader.loadNeverNullForNonNullInputOids] performs in production, just run + * [NullabilityCatalog.loadNeverNullForNonNullInputOids] performs in production, just run * per-signature) and asserts the two sets are equal. A signature this method calls a skip that * the catalog says DOES resolve is then a loud assertion failure — exactly the case a future * un-special-cased grammar-only entry would produce, closing the gap a bare `println` of the diff --git a/generator/src/test/kotlin/norm/generator/SqlCteClauseTest.kt b/generator/src/test/kotlin/norm/generator/SqlCteClauseTest.kt index 2dbdb18b..9d6c27e8 100644 --- a/generator/src/test/kotlin/norm/generator/SqlCteClauseTest.kt +++ b/generator/src/test/kotlin/norm/generator/SqlCteClauseTest.kt @@ -4,10 +4,8 @@ import assertk.assertThat import assertk.assertions.contains import assertk.assertions.hasSize import assertk.assertions.isEqualTo -import assertk.assertions.isFalse import assertk.assertions.isNotNull import assertk.assertions.isNull -import assertk.assertions.isTrue import org.junit.jupiter.api.Nested import org.junit.jupiter.api.Test @@ -16,26 +14,6 @@ class SqlCteClauseTest { @Nested inner class ParseCteClauseTest { - @Test - fun `WITH RECURSIVE is flagged as recursive`() { - val result = parseCteClause("WITH RECURSIVE counter(n) AS (SELECT 1) SELECT n FROM counter") - assertThat(result!!.isRecursive).isTrue() - } - - @Test - fun `plain WITH is not flagged as recursive`() { - val result = parseCteClause("WITH c AS (SELECT 1) SELECT * FROM c") - assertThat(result!!.isRecursive).isFalse() - } - - @Test - fun `CTE named recursive_cte is not misread as the RECURSIVE keyword`() { - // "RECURSIVE" must be matched at a word boundary — a CTE literally named "recursive_cte" - // must not cause isRecursive to be incorrectly set. - val result = parseCteClause("WITH recursive_cte AS (SELECT 1) SELECT * FROM recursive_cte") - assertThat(result!!.isRecursive).isFalse() - } - @Test fun `plain unquoted name has an identical name and rawName`() { val result = parseCteClause("WITH c AS (SELECT 1) SELECT * FROM c") diff --git a/generator/src/test/kotlin/norm/generator/SqlIdentifiersTest.kt b/generator/src/test/kotlin/norm/generator/SqlIdentifiersTest.kt index e461197c..c8ec2a44 100644 --- a/generator/src/test/kotlin/norm/generator/SqlIdentifiersTest.kt +++ b/generator/src/test/kotlin/norm/generator/SqlIdentifiersTest.kt @@ -67,4 +67,33 @@ class SqlIdentifiersTest { assertThat(truncateIdentifier("")).isEqualTo("") } } + + @Nested + inner class QuoteSqlIdentifierIfNeededSweep { + + @Test + fun `a plain lowercase identifier is returned bare`() { + assertThat(quoteSqlIdentifierIfNeeded("author", reservedWords = emptySet())).isEqualTo("author") + } + + @Test + fun `a reserved word is quoted even though it matches the bare pattern`() { + assertThat(quoteSqlIdentifierIfNeeded("order", reservedWords = setOf("order"))).isEqualTo("\"order\"") + } + + @Test + fun `an identifier with an uppercase letter is quoted`() { + assertThat(quoteSqlIdentifierIfNeeded("Foo", reservedWords = emptySet())).isEqualTo("\"Foo\"") + } + + @Test + fun `an embedded double quote is doubled per PostgreSQL's own quoted-identifier escape rule`() { + assertThat(quoteSqlIdentifierIfNeeded("a\"b", reservedWords = emptySet())).isEqualTo("\"a\"\"b\"") + } + + @Test + fun `a dollar sign after the first character is a legal bare identifier character`() { + assertThat(quoteSqlIdentifierIfNeeded("my\$col", reservedWords = emptySet())).isEqualTo("my\$col") + } + } } diff --git a/generator/src/test/kotlin/norm/generator/SqlParameterInferrerTest.kt b/generator/src/test/kotlin/norm/generator/SqlParameterInferrerTest.kt index 215f5f7a..d13c66b2 100644 --- a/generator/src/test/kotlin/norm/generator/SqlParameterInferrerTest.kt +++ b/generator/src/test/kotlin/norm/generator/SqlParameterInferrerTest.kt @@ -207,7 +207,7 @@ class SqlParameterInferrerTest { // '\(' as a real parenthesis, so extractFunctionCalls's own paren search never found a // balanced close for this call — the call was skipped entirely, and the parameter fell // through to a caller-level generic default (p1) instead of a real name. Fixed by - // SqlUtils.kt's lexical-aware findMatchingCloseParenthesis. + // SqlKeywordScanner.kt's lexical-aware findMatchingCloseParenthesis. // "string" is the correct name per Norm's own rule (see "infers formal argument names from // pg_proc" above): a pg_proc formal argument name always wins over a generic fallback, and // regexp_replace(string, pattern, replacement) is regexp_replace's real 3-argument @@ -262,7 +262,6 @@ class SqlParameterInferrerTest { inner class ResolveNullability { private val catalog = Catalog( - defaultSchema = "public", schemas = listOf( Schema( name = "public", diff --git a/generator/src/test/kotlin/norm/generator/SqlPlaceholdersTest.kt b/generator/src/test/kotlin/norm/generator/SqlPlaceholdersTest.kt index f9590b9a..0e7d0c02 100644 --- a/generator/src/test/kotlin/norm/generator/SqlPlaceholdersTest.kt +++ b/generator/src/test/kotlin/norm/generator/SqlPlaceholdersTest.kt @@ -1,7 +1,6 @@ package norm.generator import assertk.assertThat -import assertk.assertions.contains import assertk.assertions.isEqualTo import org.junit.jupiter.api.Nested import org.junit.jupiter.api.Test @@ -13,105 +12,114 @@ class SqlPlaceholdersTest { @Test fun `replaces single placeholder`() { - val result = replaceParameterPlaceholders("SELECT * FROM t WHERE id = ?") + val result = replaceParameterPlaceholders("SELECT * FROM t WHERE id = ?") { "NULL" } assertThat(result).isEqualTo("SELECT * FROM t WHERE id = NULL") } @Test fun `replaces multiple placeholders`() { - val result = replaceParameterPlaceholders("SELECT * FROM t WHERE a = ? AND b = ?") + val result = replaceParameterPlaceholders("SELECT * FROM t WHERE a = ? AND b = ?") { "NULL" } assertThat(result).isEqualTo("SELECT * FROM t WHERE a = NULL AND b = NULL") } @Test fun `preserves question mark inside single-quoted string`() { - val result = replaceParameterPlaceholders("SELECT * FROM t WHERE note = 'really?' AND id = ?") + val result = replaceParameterPlaceholders("SELECT * FROM t WHERE note = 'really?' AND id = ?") { "NULL" } assertThat(result).isEqualTo("SELECT * FROM t WHERE note = 'really?' AND id = NULL") } @Test fun `preserves question mark inside escaped string literal`() { - val result = replaceParameterPlaceholders("SELECT * FROM t WHERE note = 'it''s a ? mark' AND id = ?") + val result = + replaceParameterPlaceholders("SELECT * FROM t WHERE note = 'it''s a ? mark' AND id = ?") { "NULL" } assertThat(result).isEqualTo("SELECT * FROM t WHERE note = 'it''s a ? mark' AND id = NULL") } @Test fun `preserves question mark inside line comment`() { - val result = replaceParameterPlaceholders("SELECT * FROM t -- why?\nWHERE id = ?") + val result = replaceParameterPlaceholders("SELECT * FROM t -- why?\nWHERE id = ?") { "NULL" } assertThat(result).isEqualTo("SELECT * FROM t -- why?\nWHERE id = NULL") } @Test fun `preserves question mark inside block comment`() { - val result = replaceParameterPlaceholders("SELECT * FROM t /* what? */ WHERE id = ?") + val result = replaceParameterPlaceholders("SELECT * FROM t /* what? */ WHERE id = ?") { "NULL" } assertThat(result).isEqualTo("SELECT * FROM t /* what? */ WHERE id = NULL") } @Test fun `no placeholders returns unchanged`() { val sql = "SELECT * FROM department" - val result = replaceParameterPlaceholders(sql) + val result = replaceParameterPlaceholders(sql) { "NULL" } assertThat(result).isEqualTo(sql) } @Test fun `unclosed string literal preserves content without replacing`() { // Valid SQL never has unclosed literals, but the function should not crash - val result = replaceParameterPlaceholders("SELECT '?") + val result = replaceParameterPlaceholders("SELECT '?") { "NULL" } assertThat(result).isEqualTo("SELECT '?") } @Test fun `unclosed block comment preserves content without replacing`() { - val result = replaceParameterPlaceholders("SELECT /* ?") + val result = replaceParameterPlaceholders("SELECT /* ?") { "NULL" } assertThat(result).isEqualTo("SELECT /* ?") } - } - @Nested - inner class ReplaceParameterPlaceholdersWithSentinels { + @Test + fun `preserves question mark inside a dollar-quoted string`() { + val result = replaceParameterPlaceholders("SELECT \$\$a ? b\$\$ WHERE id = ?") { "NULL" } + assertThat(result).isEqualTo("SELECT \$\$a ? b\$\$ WHERE id = NULL") + } + + @Test + fun `preserves question mark inside a tagged dollar-quoted string`() { + val result = replaceParameterPlaceholders("SELECT \$tag\$a ? b\$tag\$ WHERE id = ?") { "NULL" } + assertThat(result).isEqualTo("SELECT \$tag\$a ? b\$tag\$ WHERE id = NULL") + } @Test - fun `replaces each placeholder with corresponding sentinel`() { - val result = replaceParameterPlaceholdersWithSentinels( - "SELECT digest(?, ?)", - listOf("'\\x00'::bytea", "''::text"), - ) - assertThat(result).isEqualTo("SELECT digest('\\x00'::bytea, ''::text)") + fun `preserves question mark inside a quoted identifier`() { + val result = replaceParameterPlaceholders("""SELECT "quoted?identifier" WHERE id = ?""") { "NULL" } + assertThat(result).isEqualTo("""SELECT "quoted?identifier" WHERE id = NULL""") } @Test - fun `falls back to NULL when sentinels exhausted`() { - val result = replaceParameterPlaceholdersWithSentinels( - "SELECT ?, ?", - listOf("0::int4"), - ) - assertThat(result).isEqualTo("SELECT 0::int4, NULL") + fun `preserves question mark inside an E-string escape sequence`() { + val result = replaceParameterPlaceholders("""SELECT E'a\'?b' WHERE id = ?""") { "NULL" } + assertThat(result).isEqualTo("""SELECT E'a\'?b' WHERE id = NULL""") } @Test - fun `skips placeholders in string literals`() { - val result = replaceParameterPlaceholdersWithSentinels( - "SELECT '?' || ?", - listOf("''::text"), - ) - assertThat(result).isEqualTo("SELECT '?' || ''::text") + fun `question mark after the inner close of a nested block comment is left alone`() { + // The old hand-rolled scanner searched for the first "*/" from the opening "/*", so it read + // this comment as ending at the inner close and would have converted the "?" that follows. + // skipLexicalToken honors PostgreSQL's documented nesting-depth semantics instead, treating + // the whole span as a single comment that only ends at the "*/" bringing the depth back to + // zero -- this is an accepted behavior change, matching PostgreSQL's own nested comments. + val result = replaceParameterPlaceholders("SELECT /* a /* b */ c ? */ 1") { "NULL" } + assertThat(result).isEqualTo("SELECT /* a /* b */ c ? */ 1") } @Test - fun `skips placeholders in line comments`() { - val result = replaceParameterPlaceholdersWithSentinels( - "SELECT -- ?\n?", - listOf("0::int4"), - ) - assertThat(result).isEqualTo("SELECT -- ?\n0::int4") + fun `replacement lambda receives 0-based parameter indices in order`() { + val observedIndices = mutableListOf() + val result = replaceParameterPlaceholders("SELECT ?, ?, ?") { index -> + observedIndices.add(index) + "\$$index" + } + assertThat(observedIndices).isEqualTo(listOf(0, 1, 2)) + assertThat(result).isEqualTo("SELECT \$0, \$1, \$2") } @Test - fun `returns original when no placeholders`() { - val sql = "SELECT 1" - val result = replaceParameterPlaceholdersWithSentinels(sql, listOf("0::int4")) - assertThat(result).isEqualTo("SELECT 1") + fun `fewer sentinels than placeholders falls back per the lambda`() { + val sentinels = listOf("0::int4") + val result = replaceParameterPlaceholders("SELECT digest(?, ?)") { index -> + sentinels.getOrElse(index) { "NULL" } + } + assertThat(result).isEqualTo("SELECT digest(0::int4, NULL)") } } diff --git a/generator/src/test/kotlin/norm/generator/TypeRepositoryTest.kt b/generator/src/test/kotlin/norm/generator/TypeRepositoryTest.kt index 5b62608c..ed0e5b8d 100644 --- a/generator/src/test/kotlin/norm/generator/TypeRepositoryTest.kt +++ b/generator/src/test/kotlin/norm/generator/TypeRepositoryTest.kt @@ -563,11 +563,11 @@ class TypeRepositoryTest { @Test fun `a reserved-word relation name is double-quoted in its table_column source reference`() { - // SQL_UNQUOTED_IDENTIFIER alone accepts "order" -- it's already all-lowercase with no special - // characters -- so without consulting the live server's own reserved-word set, the relation - // name would be rendered bare: "order.id" reads back as PostgreSQL parsing "order" as the - // reserved keyword, not a table reference (`SELECT order.id FROM "order"` fails with `syntax - // error at or near "."`). + // quoteSqlIdentifierIfNeeded's bare-identifier pattern alone accepts "order" -- it's already + // all-lowercase with no special characters -- so without consulting the live server's own + // reserved-word set, the relation name would be rendered bare: "order.id" reads back as + // PostgreSQL parsing "order" as the reserved keyword, not a table reference (`SELECT order.id + // FROM "order"` fails with `syntax error at or near "."`). val orderColumn = Column( name = "id", notNull = true, diff --git a/runtime/src/main/kotlin/norm/TransactionalConnectionProvider.kt b/runtime/src/main/kotlin/norm/TransactionalConnectionProvider.kt index 884a043e..d56589ad 100644 --- a/runtime/src/main/kotlin/norm/TransactionalConnectionProvider.kt +++ b/runtime/src/main/kotlin/norm/TransactionalConnectionProvider.kt @@ -67,11 +67,11 @@ public class TransactionalConnectionProvider(private val dataSource: DataSource) */ @Throws(SQLException::class, IllegalStateException::class) public fun transaction(readOnly: Boolean = true, body: TransactionScope.() -> Unit) { - val parent = activeTransaction.get() - if (parent != null) { - executeNestedVoid(parent, readOnly, body) - } else { - executeOutermostVoid(readOnly, body) + try { + transactionWithResult(readOnly, body) + } catch (_: RollbackException) { + // Explicit rollback already happened inside transactionWithResult; a void transaction has + // nothing to return, so the signal ends here. } } @@ -86,33 +86,13 @@ public class TransactionalConnectionProvider(private val dataSource: DataSource) public fun transactionWithResult(readOnly: Boolean = true, body: TransactionScope.() -> R): R { val parent = activeTransaction.get() return if (parent != null) { - executeNestedWithResult(parent, readOnly, body) + executeNested(parent, readOnly, body) } else { - executeOutermostWithResult(readOnly, body) + executeOutermost(readOnly, body) } } - private fun executeOutermostVoid(readOnly: Boolean, body: TransactionScope.() -> Unit) { - val connection = dataSource.connection - try { - val tx = beginOutermost(connection, readOnly) - val scope = TransactionScopeImpl() - try { - scope.body() - } catch (_: RollbackException) { - connection.rollback() - return - } catch (expected: Throwable) { - connection.rollback() - throw expected - } - commitOrRollback(tx, connection) - } finally { - cleanupOutermost(connection) - } - } - - private fun executeOutermostWithResult(readOnly: Boolean, body: TransactionScope.() -> R): R { + private fun executeOutermost(readOnly: Boolean, body: TransactionScope.() -> R): R { val connection = dataSource.connection try { val tx = beginOutermost(connection, readOnly) @@ -161,27 +141,7 @@ public class TransactionalConnectionProvider(private val dataSource: DataSource) connection.close() } - private fun executeNestedVoid(parent: Transaction, readOnly: Boolean, body: TransactionScope.() -> Unit) { - val (connection, savepoint) = beginNested(parent, readOnly) - try { - val scope = TransactionScopeImpl() - try { - scope.body() - } catch (_: RollbackException) { - connection.rollback(savepoint) - return - } catch (expected: Throwable) { - connection.rollback(savepoint) - parent.poisoned = true - throw expected - } - connection.releaseSavepoint(savepoint) - } finally { - activeTransaction.set(parent) - } - } - - private fun executeNestedWithResult(parent: Transaction, readOnly: Boolean, body: TransactionScope.() -> R): R { + private fun executeNested(parent: Transaction, readOnly: Boolean, body: TransactionScope.() -> R): R { val (connection, savepoint) = beginNested(parent, readOnly) try { val scope = TransactionScopeImpl() diff --git a/runtime/src/test/kotlin/norm/TransactionalConnectionProviderTest.kt b/runtime/src/test/kotlin/norm/TransactionalConnectionProviderTest.kt index b890b5f3..bd64604e 100644 --- a/runtime/src/test/kotlin/norm/TransactionalConnectionProviderTest.kt +++ b/runtime/src/test/kotlin/norm/TransactionalConnectionProviderTest.kt @@ -4,6 +4,7 @@ import assertk.assertThat import assertk.assertions.contains import assertk.assertions.isEqualTo import assertk.assertions.isNotNull +import assertk.assertions.isTrue import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Nested import org.junit.jupiter.api.Test @@ -14,6 +15,7 @@ import org.testcontainers.containers.PostgreSQLContainer import org.testcontainers.containers.wait.strategy.Wait import org.testcontainers.junit.jupiter.Container import org.testcontainers.junit.jupiter.Testcontainers +import java.sql.Connection import java.sql.DriverManager import java.util.concurrent.ConcurrentHashMap import javax.sql.DataSource @@ -102,6 +104,28 @@ class TransactionalConnectionProviderTest { val count = countRows() assertThat(count).isEqualTo(0) } + + @Test + fun `outermost explicit rollback — connection rolled back once, autoCommit restored, and closed`() { + val recordingDataSource = RecordingDataSource(postgres.jdbcUrl, postgres.username, postgres.password) + val recordingProvider = TransactionalConnectionProvider(recordingDataSource) + + recordingProvider.transaction(readOnly = false) { + recordingProvider.withConnection { conn -> + conn.prepareStatement("INSERT INTO test_data (value) VALUES (?)").use { stmt -> + stmt.setString(1, "should-not-persist") + stmt.executeUpdate() + } + } + rollback() + } + + assertThat(recordingDataSource.connections.size).isEqualTo(1) + val connection = recordingDataSource.connections.single() + assertThat(connection.rollbackCount).isEqualTo(1) + assertThat(connection.lastAutoCommit).isEqualTo(true) + assertThat(connection.closed).isTrue() + } } @Nested @@ -332,4 +356,69 @@ class TransactionalConnectionProviderTest { override fun getLoginTimeout(): Int = throw UnsupportedOperationException() override fun getParentLogger() = throw UnsupportedOperationException() } + + /** + * A real JDBC connection that records [rollback], [setAutoCommit], and [close] calls so tests can + * assert on connection lifecycle without mocking. All other operations delegate unchanged to the + * underlying connection. + */ + private class RecordingConnection(private val delegate: Connection) : Connection by delegate { + var rollbackCount: Int = 0 + private set + var lastAutoCommit: Boolean? = null + private set + var closed: Boolean = false + private set + + override fun rollback() { + rollbackCount++ + delegate.rollback() + } + + override fun setAutoCommit(autoCommit: Boolean) { + lastAutoCommit = autoCommit + delegate.setAutoCommit(autoCommit) + } + + override fun close() { + closed = true + delegate.close() + } + + override fun isClosed(): Boolean = closed + } + + /** + * A [DataSource] that hands out real [DriverManager] connections wrapped in [RecordingConnection], + * keeping every connection it created so tests can assert on their recorded lifecycle. + */ + private class RecordingDataSource( + private val url: String, + private val username: String, + private val password: String, + ) : DataSource { + val connections: MutableList = mutableListOf() + + override fun getConnection(): Connection { + val connection = RecordingConnection(DriverManager.getConnection(url, username, password)) + connections.add(connection) + return connection + } + + override fun getConnection(username: String?, password: String?): Connection { + val connection = RecordingConnection( + DriverManager.getConnection(url, username ?: this.username, password ?: this.password), + ) + connections.add(connection) + return connection + } + + override fun unwrap(iface: Class?): T = throw UnsupportedOperationException() + override fun isWrapperFor(iface: Class<*>?): Boolean = false + override fun getLogWriter() = throw UnsupportedOperationException() + override fun setLogWriter(out: java.io.PrintWriter?) = throw UnsupportedOperationException() + override fun setLoginTimeout(seconds: Int) = throw UnsupportedOperationException() + override fun getLoginTimeout(): Int = throw UnsupportedOperationException() + override fun getParentLogger() = throw UnsupportedOperationException() + } }