expand parser with live views, row expiry, posting indexes

src/autocomplete/content-assist.ts

@@ -7,7 +7,10 @@ import {
   IDENTIFIER_KEYWORD_TOKENS,
   EXPRESSION_OPERATORS,
 } from "./token-classification"
-import { computeContentAssistBudgeted } from "./budgeted-content-assist"
+import {
+  computeContentAssistBudgeted,
+  type ContentAssistSuggestion,
+} from "./budgeted-content-assist"
 
 // =============================================================================
 // Constants
@@ -99,6 +102,14 @@ export interface ContentAssistResult {
   referencedColumns: Set<string>
   /** Whether the cursor is inside a WHERE clause expression */
   isConditionContext: boolean
+  /**
+   * Concrete keywords the grammar accepts only through the generic `identifier`
+   * sub-rule (so the word stays non-reserved) and therefore never surface via
+   * the normal `IdentifierKeyword` path. Re-injected per rule context so
+   * multi-word `GRANT`/`REVOKE`, `CONVERT PARTITION` and `SHOW CREATE DATABASE`
+   * clauses can be autocompleted. See `contextKeywordSuggestions`.
+   */
+  contextKeywords: string[]
 }
 
 // =============================================================================
@@ -829,6 +840,86 @@ function accumulateFlags(target: CategoryFlags, kind: PositionKind): void {
 interface ComputeResult extends CategoryFlags {
   nextTokenTypes: TokenType[]
   isConditionContext: boolean
+  contextKeywords: string[]
+}
+
+// Category names valid inside SHOW CREATE DATABASE (INCLUDE|EXCLUDE) ( ... ).
+// The grammar consumes these via `identifier` (they are non-reserved), so only
+// the explicit `ALL` token surfaces on its own; re-inject the rest.
+const DATABASE_CATEGORY_KEYWORDS = [
+  "ALL",
+  "TABLES",
+  "VIEWS",
+  "MATERIALIZED_VIEWS",
+  "USERS",
+  "GROUPS",
+  "SERVICE_ACCOUNTS",
+  "PERMISSIONS",
+  "SCHEMA",
+  "ACL",
+]
+
+/**
+ * Re-inject the concrete keywords that the grammar only accepts through the
+ * generic `identifier` sub-rule — kept non-reserved on purpose, which means
+ * Chevrotain's content-assist reports them merely as the abstract
+ * `IdentifierKeyword` category and the suggestion builder renders that slot as
+ * "type a name here" rather than a keyword. We recover the intended words from
+ * the active rule context (the `ruleStack` of any `IdentifierKeyword` path) plus
+ * the immediately preceding tokens, so multi-word `GRANT`/`REVOKE`,
+ * `CONVERT PARTITION` and `SHOW CREATE DATABASE` clauses autocomplete
+ * token-by-token. Parsing already accepts all of these; this only affects hints.
+ */
+function contextKeywordSuggestions(
+  tokens: IToken[],
+  suggestions: ContentAssistSuggestion[],
+): string[] {
+  // Rule contexts in which the `identifier` catch-all is a valid next token.
+  const idKeywordStacks = suggestions
+    .filter((s) => s.nextTokenType.name === "IdentifierKeyword")
+    .map((s) => s.ruleStack)
+  if (idKeywordStacks.length === 0) return []
+  const inRule = (r: string): boolean =>
+    idKeywordStacks.some((stack) => stack.includes(r))
+
+  const last = tokens[tokens.length - 1]?.tokenType.name
+  const prev = tokens[tokens.length - 2]?.tokenType.name
+  const out = new Set<string>()
+
+  // GRANT / REVOKE permission phrases whose words route through `identifier`:
+  //   CONVERT PARTITION [TO PARQUET|NATIVE], SET TABLE FORMAT, SET TABLE TYPE,
+  //   SET STORAGE POLICY, REMOVE STORAGE POLICY.
+  // (SET / TABLE / TO / PARQUET / NATIVE at their spots are explicit CONSUMEs
+  // and already surface; we only add the identifier-path words.)
+  if (inRule("permissionToken")) {
+    if (last === "Grant" || last === "Revoke") {
+      out.add("CONVERT")
+      out.add("REMOVE")
+    } else if (last === "Convert") {
+      out.add("PARTITION")
+    } else if (last === "Set" || last === "Remove") {
+      out.add("STORAGE")
+    } else if (last === "Table" && prev === "Set") {
+      out.add("FORMAT")
+      out.add("TYPE")
+    } else if (last === "Storage" && (prev === "Set" || prev === "Remove")) {
+      out.add("POLICY")
+    }
+  }
+
+  // ALTER TABLE ... CONVERT PARTITION TO { PARQUET | NATIVE }: the target is
+  // OR([CONSUME(Table), SUBRULE(identifier)]) so PARQUET/NATIVE go via identifier.
+  if (inRule("convertPartitionTarget") && last === "To") {
+    out.add("PARQUET")
+    out.add("NATIVE")
+  }
+
+  // SHOW CREATE DATABASE (INCLUDE|EXCLUDE) ( <category>, ... ).
+  if (inRule("showCreateDatabaseCategory")) {
+    for (const c of DATABASE_CATEGORY_KEYWORDS) out.add(c)
+  }
+
+  return [...out]
 }
 
 /**
@@ -907,10 +998,13 @@ function computeSuggestions(tokens: IToken[]): ComputeResult {
       s.ruleStack.includes("whereClause"),
   )
 
+  const contextKeywords = contextKeywordSuggestions(tokens, suggestions)
+
   return {
     nextTokenTypes: result,
     ...flags,
     isConditionContext,
+    contextKeywords,
   }
 }
 
@@ -1051,6 +1145,7 @@ export function getContentAssist(
         suggestTableValuedFunctions: false,
         referencedColumns: new Set(),
         isConditionContext: false,
+        contextKeywords: [],
       }
     }
   }
@@ -1084,6 +1179,7 @@ export function getContentAssist(
   let suggestWindowFunctions = false
   let suggestTableValuedFunctions = false
   let isConditionContext = false
+  let contextKeywords: string[] = []
   try {
     const computed = computeSuggestions(tokensForAssist)
     nextTokenTypes = computed.nextTokenTypes
@@ -1094,6 +1190,7 @@ export function getContentAssist(
     suggestWindowFunctions = computed.suggestWindowFunctions
     suggestTableValuedFunctions = computed.suggestTableValuedFunctions
     isConditionContext = computed.isConditionContext
+    contextKeywords = computed.contextKeywords
   } catch (e) {
     // If content assist fails, return empty suggestions
     // This can happen with malformed input
@@ -1185,6 +1282,7 @@ export function getContentAssist(
     suggestTableValuedFunctions,
     referencedColumns,
     isConditionContext,
+    contextKeywords,
   }
 }
 

src/autocomplete/provider.ts

@@ -194,6 +194,7 @@ export function createAutocompleteProvider(
         suggestTableValuedFunctions,
         referencedColumns,
         isConditionContext,
+        contextKeywords,
       } = getContentAssist(query, cursorOffset)
 
       // Merge CTE columns into the schema so getColumnsInScope() can find them
@@ -264,6 +265,28 @@ export function createAutocompleteProvider(
             partialPrefix,
           },
         )
+        if (contextKeywords.length > 0) {
+          const seen = new Set(suggestions.map((s) => s.label.toUpperCase()))
+          for (const kw of contextKeywords) {
+            if (seen.has(kw.toUpperCase())) continue
+            if (
+              isMidWord &&
+              partialPrefix &&
+              !kw.toLowerCase().startsWith(partialPrefix)
+            ) {
+              continue
+            }
+            seen.add(kw.toUpperCase())
+            suggestions.push({
+              label: kw,
+              kind: SuggestionKind.Keyword,
+              insertText: kw,
+              filterText: kw.toLowerCase(),
+              priority: SuggestionPriority.Medium,
+            })
+          }
+        }
+
         if (suggestTables) {
           rankTableSuggestions(suggestions, referencedColumns, columnIndex)
         }

src/grammar/constants.ts

@@ -1,21 +1,29 @@
 export const constants: string[] = [
+  "acl",
   "asc",
+  "beginning",
+  "bitmap",
   "brotli",
   "century",
   "complete",
+  "daily",
   "datestyle",
   "day",
   "days",
   "decade",
   "default_transaction_read_only",
+  "delta",
   "delta_binary_packed",
   "delta_length_byte_array",
   "desc",
   "dow",
   "doy",
+  "ef",
   "epoch",
+  "expression",
   "false",
   "gzip",
+  "highest",
   "hour",
   "hours",
   "http",
@@ -27,10 +35,12 @@ export const constants: string[] = [
   "jwk",
   "linear",
   "local",
+  "lowest",
   "lz4",
   "lz4_raw",
   "lzo",
   "manual",
+  "materialized_views",
   "max_identifier_length",
   "microsecond",
   "microseconds",
@@ -46,28 +56,34 @@ export const constants: string[] = [
   "nanoseconds",
   "native",
   "none",
+  "now",
   "null",
   "parquet",
   "pgwire",
   "plain",
+  "posting",
   "prepare",
   "prev",
   "quarter",
   "remote",
   "rest",
   "rle_dictionary",
+  "schema",
   "search_path",
   "second",
   "seconds",
   "server_version",
   "server_version_num",
+  "service_accounts",
   "skip_column",
   "skip_row",
   "snappy",
   "standard_conforming_strings",
+  "stats",
   "transaction_isolation",
   "true",
   "uncompressed",
+  "views",
   "week",
   "weeks",
   "year",

src/grammar/functions.ts

@@ -16,8 +16,8 @@
 //   letter "p" alone doesn't bury real schema results in JOIN positions.
 //   `information_schema.*` is NOT gated — those names are SQL-standard.
 //
-// Removed from earlier flat list: 17 entries absent from QuestDB runtime AND
-// docs (e.g. `array_agg`, `lcase`, `len`, `nvl`, `headers`, `show`,
+// Removed from earlier flat list: entries absent from QuestDB runtime AND
+// docs (e.g. `lcase`, `len`, `nvl`, `headers`, `show`,
 // `commitLag`, `batch`); 11 SQL operators / syntactic keywords (e.g. `and`,
 // `or`, `between`, `case`, `cast`) — they exist as factories internally but
 // users always write them as syntax, and the parser keyword path covers them.
@@ -84,6 +84,7 @@ export const scalarFunctions: string[] = [
   "insertion_point",
   "interval_end",
   "interval_start",
+  "is_end_of_month",
   "is_leap_year",
   "json_extract",
   "l2price",
@@ -229,6 +230,7 @@ export const aggregateFunctions: string[] = [
   "approx_percentile",
   "arg_max",
   "arg_min",
+  "array_agg",
   "array_elem_avg",
   "array_elem_max",
   "array_elem_min",
@@ -250,14 +252,21 @@ export const aggregateFunctions: string[] = [
   "haversine_dist_deg",
   "isOrdered",
   "ksum",
+  "kurtosis",
+  "kurtosis_pop",
+  "kurtosis_samp",
   "last",
   "last_not_null",
   "max",
   "min",
   "mode",
   "nsum",
   "regr_intercept",
+  "regr_r2",
   "regr_slope",
+  "skewness",
+  "skewness_pop",
+  "skewness_samp",
   "sparkline",
   "stddev",
   "stddev_pop",
@@ -277,11 +286,14 @@ export const aggregateFunctions: string[] = [
 ]
 
 export const windowFunctions: string[] = [
+  "cume_dist",
   "dense_rank",
   "first_value",
   "lag",
   "last_value",
   "lead",
+  "nth_value",
+  "ntile",
   "percent_rank",
   "rank",
   "row_number",
@@ -290,6 +302,7 @@ export const windowFunctions: string[] = [
 export const tableValuedFunctions: string[] = [
   // CURSOR type — explicitly row-returning
   "all_permissions",
+  "backups",
   "generate_series",
   "information_schema._pg_expandarray",
   "long_sequence",
@@ -313,13 +326,15 @@ export const tableValuedFunctions: string[] = [
   "pg_roles",
   "pg_type",
   "read_parquet",
+  "sleep",
   "table_partitions",
   // STANDARD type but succeed as `SELECT * FROM <name>()` — meta pseudo-tables
   "all_tables",
   "export_files",
   "functions",
   "import_files",
   "keywords",
+  "live_views",
   "materialized_views",
   "memory_metrics",
   "permissions",