Incremental caching support for large codebases

.eslintrc.json

@@ -1,4 +1,5 @@
 {
+  "root": true,
   "env": {
     "browser": true,
     "node": true

.mocharc.cjs

@@ -1,6 +1,6 @@
 const fs = require('fs')
 const ignore = ['test/language/testdata/**/*']
-if (!fs.existsSync('node_modules/tree-sitter-java')) {
+if (!fs.existsSync('node_modules/tree-sitter')) {
   ignore.push('test/language/ExpressionBuilder.test.*')
   ignore.push('test/service/snippet/stepDefinitionSnippet.test.*')
 }

.mocharc.cjs.cjs

@@ -1,6 +1,6 @@
 const fs = require('fs')
 const ignore = ['dist/cjs/test/language/testdata/**/*']
-if (!fs.existsSync('node_modules/tree-sitter-java')) {
+if (!fs.existsSync('node_modules/tree-sitter')) {
   ignore.push('dist/cjs/test/language/ExpressionBuilder.test.*')
   ignore.push('dist/cjs/test/service/snippet/stepDefinitionSnippet.test.*')
 }

.tool-versions

@@ -0,0 +1 @@
+nodejs 24.11.0

package.json

@@ -7,10 +7,10 @@
   "module": "dist/esm/src/index.js",
   "types": "dist/esm/src/index.d.ts",
   "engines": {
-    "node": "18 || 20 || 22 || >=23"
+    "node": "18 || 20 || 22 || >=23 || >=24"
   },
   "enginesTested": {
-    "node": "18 || 20 || 22 || 23"
+    "node": "18 || 20 || 22 || 23 || 24"
   },
   "exports": {
     ".": {

src/language/ExpressionBuilder.ts

@@ -4,7 +4,7 @@ import {
   ParameterTypeRegistry,
 } from '@cucumber/cucumber-expressions'
 
-import { createLocationLink, makeParameterType, sortLinks } from './helpers.js'
+import { createLocationLink, makeParameterType } from './helpers.js'
 import { SourceAnalyzer } from './SourceAnalyzer.js'
 import {
   ExpressionBuilderResult,
@@ -41,28 +41,96 @@ export class ExpressionBuilder {
 
     const sourceAnalyser = new SourceAnalyzer(this.parserAdapter, sources)
 
+    const parameterTypeLinks: Map<string, ParameterTypeLink[]> = new Map()
+    sourceAnalyser.eachParameterTypeLink((parameterTypeLink, source) => {
+      defineParameterType(parameterTypeLink.parameterType)
+      if (!parameterTypeLinks.has(source.uri)) {
+        parameterTypeLinks.set(source.uri, [])
+      }
+      parameterTypeLinks.get(source.uri)?.push(parameterTypeLink)
+    })
+
+    const expressionLinks: Map<string, ExpressionLink[]> = new Map()
+    sourceAnalyser.eachStepDefinitionExpression(
+      (stepDefinitionExpression, rootNode, expressionNode, source) => {
+        try {
+          const expression = expressionFactory.createExpression(stepDefinitionExpression)
+          const locationLink = createLocationLink(rootNode, expressionNode, source.uri)
+          if (!expressionLinks.has(source.uri)) {
+            expressionLinks.set(source.uri, [])
+          }
+          expressionLinks.get(source.uri)?.push({ expression, locationLink })
+        } catch (err) {
+          errors.push(err)
+        }
+      }
+    )
+
+    return {
+      newExpressionLinks: new Map(), // during initial build keep empty
+      expressionLinks,
+      parameterTypeLinks,
+      errors: sourceAnalyser.getErrors().concat(errors),
+      registry,
+    }
+  }
+
+  /**
+   * Rebuilds the expression builder result with new sources
+   * @param existingResult - The existing expression builder result
+   * @param sources - The new sources to update the expression builder result with
+   * @returns The updated expression builder result
+   */
+  rebuild(
+    existingResult: ExpressionBuilderResult,
+    sources: readonly Source<LanguageName>[]
+  ): ExpressionBuilderResult {
+    const errors: Error[] = []
+    const registry = existingResult.registry
+    const expressionFactory = new ExpressionFactory(registry)
+    const newExpressionLinks: Map<string, ExpressionLink[]> = new Map()
+
+    const sourceAnalyser = new SourceAnalyzer(this.parserAdapter, sources)
+
+    // redefine is not allowed, so we ignore the error
+    function defineParameterType(parameterType: ParameterType<unknown>) {
+      try {
+        registry.defineParameterType(parameterType)
+      } catch (err) {
+        // ignore error
+      }
+    }
+
     const parameterTypeLinks: ParameterTypeLink[] = []
     sourceAnalyser.eachParameterTypeLink((parameterTypeLink) => {
       defineParameterType(parameterTypeLink.parameterType)
       parameterTypeLinks.push(parameterTypeLink)
     })
 
-    const expressionLinks: ExpressionLink[] = []
+    const cleared = new Map<string, boolean>()
     sourceAnalyser.eachStepDefinitionExpression(
       (stepDefinitionExpression, rootNode, expressionNode, source) => {
         try {
           const expression = expressionFactory.createExpression(stepDefinitionExpression)
           const locationLink = createLocationLink(rootNode, expressionNode, source.uri)
-          expressionLinks.push({ expression, locationLink })
+          // clear the existing expression links for this source
+          if (!cleared.get(source.uri)) {
+            existingResult.expressionLinks.set(source.uri, [])
+            newExpressionLinks.set(source.uri, [])
+            cleared.set(source.uri, true)
+          }
+          existingResult.expressionLinks.get(source.uri)?.push({ expression, locationLink })
+          newExpressionLinks.get(source.uri)?.push({ expression, locationLink })
         } catch (err) {
           errors.push(err)
         }
       }
     )
 
     return {
-      expressionLinks: sortLinks(expressionLinks),
-      parameterTypeLinks: sortLinks(parameterTypeLinks),
+      newExpressionLinks,
+      expressionLinks: existingResult.expressionLinks,
+      parameterTypeLinks: existingResult.parameterTypeLinks,
       errors: sourceAnalyser.getErrors().concat(errors),
       registry,
     }

src/language/SourceAnalyzer.ts

@@ -31,7 +31,9 @@ export class SourceAnalyzer {
     private readonly sources: readonly Source<LanguageName>[]
   ) {}
 
-  eachParameterTypeLink(callback: (parameterTypeLink: ParameterTypeLink) => void) {
+  eachParameterTypeLink(
+    callback: (parameterTypeLink: ParameterTypeLink, source: Source<LanguageName>) => void
+  ) {
     const parameterTypeMatches = this.getSourceMatches(
       (language: Language) => language.defineParameterTypeQueries
     )
@@ -50,11 +52,11 @@ export class SourceAnalyzer {
           const locationLink = createLocationLink(rootNode, selectionNode, source.uri)
           const props: ParameterTypeLinkProps = (propsByName[parameterTypeName] = propsByName[
             parameterTypeName
-          ] || { locationLink, regexpsList: [] })
+          ] || { locationLink, regexpsList: [], source })
           props.regexpsList.push(regExps)
         }
       }
-      for (const [name, { regexpsList, locationLink }] of Object.entries(propsByName)) {
+      for (const [name, { regexpsList, locationLink, source }] of Object.entries(propsByName)) {
         const regexps: StringOrRegExp[] = regexpsList.reduce<StringOrRegExp[]>((prev, current) => {
           if (Array.isArray(current)) {
             return prev.concat(...current)
@@ -64,7 +66,7 @@ export class SourceAnalyzer {
         }, [])
         const parameterType = makeParameterType(name, regexps)
         const parameterTypeLink: ParameterTypeLink = { parameterType, locationLink }
-        callback(parameterTypeLink)
+        callback(parameterTypeLink, source)
       }
     }
   }
@@ -143,6 +145,7 @@ language: ${source.languageName}
 }
 
 type ParameterTypeLinkProps = {
+  source: Source<LanguageName>
   regexpsList: RegExps[]
   locationLink: LocationLink
 }

src/language/helpers.ts

@@ -1,7 +1,15 @@
 import { ParameterType, RegExps } from '@cucumber/cucumber-expressions'
 import { DocumentUri, LocationLink, Range } from 'vscode-languageserver-types'
 
-import { Link, NodePredicate, TreeSitterQueryMatch, TreeSitterSyntaxNode } from './types'
+import {
+  ExpressionBuilderResult,
+  ExpressionLink,
+  Link,
+  NodePredicate,
+  ParameterTypeLink,
+  TreeSitterQueryMatch,
+  TreeSitterSyntaxNode,
+} from './types'
 
 export function syntaxNode(match: TreeSitterQueryMatch, name: string): TreeSitterSyntaxNode | null {
   const nodes = syntaxNodes(match, name)
@@ -18,14 +26,22 @@ export function makeParameterType(name: string, regexps: RegExps) {
   return new ParameterType(name, regexps, Object, (arg) => arg, true, false)
 }
 
-export function sortLinks<L extends Link>(links: L[]): readonly L[] {
+export function sortLinks<L extends Link>(links: L[]): L[] {
   return links.sort((a, b) => {
     const pathComparison = a.locationLink.targetUri.localeCompare(b.locationLink.targetUri)
     if (pathComparison !== 0) return pathComparison
     return a.locationLink.targetRange.start.line - b.locationLink.targetRange.start.line
   })
 }
 
+export function parameterTypeLinks(existingResult: ExpressionBuilderResult): ParameterTypeLink[] {
+  return sortLinks(Array.from(existingResult.parameterTypeLinks.values()).flat())
+}
+
+export function expressionLinks(existingResult: ExpressionBuilderResult): ExpressionLink[] {
+  return sortLinks(Array.from(existingResult.expressionLinks.values()).flat())
+}
+
 export function createLocationLink(
   rootNode: TreeSitterSyntaxNode,
   selectionNode: TreeSitterSyntaxNode,

src/language/index.ts

@@ -1,2 +1,3 @@
 export * from './ExpressionBuilder.js'
+export { expressionLinks, parameterTypeLinks } from './helpers.js'
 export * from './types.js'

src/language/types.ts

@@ -68,9 +68,10 @@ export type Language = Readonly<{
 }>
 
 export type ExpressionBuilderResult = Readonly<{
-  readonly expressionLinks: readonly ExpressionLink[]
-  readonly parameterTypeLinks: readonly ParameterTypeLink[]
-  readonly errors: readonly Error[]
+  readonly newExpressionLinks: Map<string, ExpressionLink[]>
+  readonly expressionLinks: Map<string, ExpressionLink[]>
+  readonly parameterTypeLinks: Map<string, ParameterTypeLink[]>
+  readonly errors: Error[]
   readonly registry: ParameterTypeRegistry
 }>
 

src/messages/MessagesBuilder.ts

@@ -7,7 +7,7 @@ import {
 import { Envelope, StepDefinitionPatternType } from '@cucumber/messages'
 
 import { extractStepTexts } from '../gherkin/extractStepTexts.js'
-import { buildSuggestions } from '../suggestions/buildSuggestions.js'
+import { buildSuggestions, sortSuggestions } from '../suggestions/buildSuggestions.js'
 import { Suggestion } from '../suggestions/types.js'
 
 export type MessagesBuilderResult = {
@@ -64,7 +64,15 @@ export class MessagesBuilder {
 
   build(): MessagesBuilderResult {
     return {
-      suggestions: buildSuggestions(this.parameterTypeRegistry, this.stepTexts, this.expressions),
+      suggestions: sortSuggestions(
+        buildSuggestions(
+          this.parameterTypeRegistry,
+          new Set(this.stepTexts),
+          this.expressions,
+          new Map(),
+          false
+        )
+      ),
       expressions: this.expressions,
     }
   }