performance related fixes

CHANGELOG.md

@@ -5,6 +5,40 @@ All notable changes to the Health Intersections Node Server will be documented i
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.11.1] - 2026-07-30
+
+### Added
+
+- Cooperative yielding for long terminology operations: after every 25ms of continuous compute the
+  operation yields the event loop (`checkAndYield`), so a heavy $expand can no longer stall every
+  other request on the server. Operations whose client has disconnected are aborted at their next
+  yield point instead of computing a response nobody will read
+- The operation time limit is now configurable (`operationTimeLimit`, seconds) and defaults to 20s
+  (below common client timeouts, so clients see the too-costly OperationOutcome rather than a
+  socket timeout), and is measured against compute time consumed rather than wall-clock, so
+  concurrent operations time-sharing the event loop don't abort each other
+- The Node Blocking chart on the status page now shows the worst single event-loop stall per
+  window as well as the mean
+
+### Fixed
+
+- fix whole-code-system SNOMED CT expansions blocking the server for 30 seconds and then failing:
+  the hierarchy walk read a non-existent `.code` property, so every concept was treated as a
+  duplicate of the first and no expansion size guard could ever stop the walk. This is what caused
+  the rash of client timeouts on tx.fhir.org after the 0.11.0 upgrade
+- the hierarchy walk now visits each concept once rather than once per path to it (SNOMED CT is a
+  poly-hierarchy: the old walk made ~15.8M visits for ~520k concepts)
+- restore the up-front too-costly refusal for expansions that cannot fit the expansion limit (the
+  totalCount size guard was silently dead - method read as a property); expansions with a
+  count/offset window inside the limit still succeed as partial, unclosed expansions
+- re-enable the CPT 1000-code expansion cap (same method-read-as-property bug)
+- fix parameter misalignment in the no-details expansion path (19 arguments passed to a
+  20-parameter function)
+
+### Tx Conformance Statement
+
+FHIRsmith passed all 2729 HL7 terminology service tests (modes tx.fhir.org+omop+general+snomed, tests v1.9.2, runner v6.9.12)
+
 ## [0.11.0] - 2026-mm-dd
 
 ### Added

stats.js

@@ -37,6 +37,10 @@ class ServerStats {
       const requestsPerMin = minutesSinceStart > 0 ? requestsDelta / minutesSinceStart : 0;
 
       const loopDelay = this.eventLoopMonitor.mean / 1e6;
+      // Worst single stall in the window - the mean dilutes a multi-second
+      // event-loop block into noise; the max is what shows request-freezing
+      // stalls (see the 2026-07-30 tx.fhir.org incident).
+      const loopMax = this.eventLoopMonitor.max / 1e6;
       // Two distinct caches: the expansion cache (count entries) and the client
       // (resource) cache (count concepts held - a sense of how much it's carrying).
       let expansionItems = 0;
@@ -50,7 +54,7 @@ class ServerStats {
         }
       }
 
-      this.history.push({time: now, mem: currentMem - this.startMem, rpm: requestsPerMin, tat: requestsTat, block: loopDelay, expansion: expansionItems, clientConcepts: clientConcepts});
+      this.history.push({time: now, mem: currentMem - this.startMem, rpm: requestsPerMin, tat: requestsTat, block: loopDelay, blockMax: loopMax, expansion: expansionItems, clientConcepts: clientConcepts});
 
       this.eventLoopMonitor.reset();
       this.requestCountSnapshot = combinedCount;

tests/tx/expand-snomed-whole.test.js

@@ -0,0 +1,173 @@
+/**
+ * Whole-code-system SNOMED CT expansion tests
+ *
+ * Regression tests for the 0.11.0 tx.fhir.org incident: expanding a value set
+ * that includes all of SNOMED CT (e.g. http://hl7.org/fhir/ValueSet/questionnaire-answers)
+ * walked every *path* of the SNOMED poly-hierarchy with `context.code` undefined
+ * for every concept, so the expansion collapsed onto a single map key, no size
+ * guard could ever fire, and each request blocked the event loop for the full
+ * 30-second deadCheck limit - timing out every other request on the server.
+ *
+ * These tests expand an include-all-of-SNOMED value set against the committed
+ * SNOMED test subset and pin the fixed behaviour:
+ *  - enumeration returns real, distinct codes (not undefined)
+ *  - the expansion is marked unclosed (grammar system)
+ *  - a count/offset window inside the limit succeeds as a partial expansion
+ *  - an expansion that cannot fit the limit is refused as too-costly up front
+ */
+
+const fs = require('fs');
+const path = require('path');
+const express = require('express');
+const cors = require('cors');
+const request = require('supertest');
+const TXModule = require('../../tx/tx');
+const folders = require('../../library/folder-setup');
+
+const SNOMED_URI = 'http://snomed.info/sct';
+
+// The whole point of these tests is that expansion terminates promptly; if any
+// of them takes anywhere near this long, the walk is broken again.
+jest.setTimeout(60000);
+
+function allOfSnomedValueSet() {
+  return {
+    resourceType: 'ValueSet',
+    url: 'http://example.org/fhir/ValueSet/all-of-snomed',
+    status: 'active',
+    compose: {
+      include: [{ system: SNOMED_URI }]
+    }
+  };
+}
+
+function expandRequest(app, params = [], headers = {}) {
+  let req = request(app)
+    .post('/tx/r5/ValueSet/$expand')
+    .set('Accept', 'application/json')
+    .set('Content-Type', 'application/json');
+  for (const [k, v] of Object.entries(headers)) {
+    req = req.set(k, v);
+  }
+  return req.send({
+    resourceType: 'Parameters',
+    parameter: [
+      { name: 'valueSet', resource: allOfSnomedValueSet() },
+      ...params
+    ]
+  });
+}
+
+function hasUnclosedExtension(expansion) {
+  return (expansion.extension || []).some(
+    e => e.url === 'http://hl7.org/fhir/StructureDefinition/valueset-unclosed');
+}
+
+describe('Whole-code-system SNOMED expansion', () => {
+  let app;
+  let txModule;
+
+  beforeAll(async () => {
+    // The library resolves source files against the terminology cache folder;
+    // put the committed test subset there under the name the fixture uses.
+    const committed = path.resolve(__dirname, '../../tx/data/snomed-testing.cache');
+    const cacheFolder = folders.subDir('terminology-cache');
+    const target = path.join(cacheFolder, 'snomed-testing.cache');
+    if (!fs.existsSync(target)) {
+      fs.copyFileSync(committed, target);
+    }
+
+    app = express();
+    app.use(cors());
+    app.use(express.json());
+    app.use(express.urlencoded({ extended: true }));
+
+    txModule = new TXModule();
+    await txModule.initialize({
+      librarySource: path.join(__dirname, 'fixtures', 'test-library-snomed.yaml'),
+      endpoints: [
+        { path: '/tx/r5', fhirVersion: '5.0', context: null }
+      ]
+    }, app);
+  }, 60000);
+
+  afterAll(async () => {
+    if (txModule) {
+      await txModule.shutdown();
+    }
+  });
+
+  test('count=10 returns 10 real, distinct codes and is marked unclosed', async () => {
+    const response = await expandRequest(app, [{ name: 'count', valueInteger: 10 }]);
+
+    expect(response.status).toBe(200);
+    expect(response.body.resourceType).toBe('ValueSet');
+    const expansion = response.body.expansion;
+    expect(expansion).toBeDefined();
+    expect(expansion.contains).toHaveLength(10);
+
+    const codes = new Set();
+    for (const c of expansion.contains) {
+      // Before the fix every entry had code undefined (context.code on a
+      // SnomedExpressionContext), which collapsed the expansion to one entry.
+      expect(typeof c.code).toBe('string');
+      expect(c.code).toMatch(/^\d+$/);
+      expect(c.system).toBe(SNOMED_URI);
+      expect(c.display).toBeTruthy();
+      codes.add(c.code);
+    }
+    expect(codes.size).toBe(10);
+
+    // SNOMED is a grammar system: an enumeration of its precoordinated concepts
+    // is inherently incomplete, so the expansion must be marked unclosed.
+    expect(hasUnclosedExtension(expansion)).toBe(true);
+  });
+
+  test('full enumeration of the subset returns every concept exactly once', async () => {
+    const response = await expandRequest(app, []);
+
+    expect(response.status).toBe(200);
+    const expansion = response.body.expansion;
+    expect(expansion).toBeDefined();
+
+    // The committed subset holds ~2k concepts - comfortably under the test
+    // limit, so the whole thing enumerates. Every code must be real and, with
+    // the visited-set fix, appear exactly once even though the subset is a
+    // poly-hierarchy (the old walk visited concepts once per path to them).
+    const codes = new Set();
+    for (const c of expansion.contains) {
+      expect(typeof c.code).toBe('string');
+      expect(c.code).toMatch(/^\d+$/);
+      codes.add(c.code);
+    }
+    expect(codes.size).toBe(expansion.contains.length);
+    expect(codes.size).toBeGreaterThan(1000);
+    expect(hasUnclosedExtension(expansion)).toBe(true);
+  });
+
+  test('a count window inside a small limit succeeds as a partial expansion', async () => {
+    // Client-supplied threshold of 50 << subset size: enumeration must stop at
+    // the limit and return the requested window rather than walking on.
+    const response = await expandRequest(app, [{ name: 'count', valueInteger: 10 }],
+      { 'x-too-costly-threshold': '50' });
+
+    expect(response.status).toBe(200);
+    const expansion = response.body.expansion;
+    expect(expansion.contains).toHaveLength(10);
+    for (const c of expansion.contains) {
+      expect(typeof c.code).toBe('string');
+      expect(c.code).toMatch(/^\d+$/);
+    }
+  });
+
+  test('an expansion that cannot fit the limit is refused as too-costly', async () => {
+    // No count, threshold below the subset size: enumeration could only ever
+    // end in too-costly at the limit, so the request must be refused up front
+    // (this used to be dead code - cs.totalCount read a method as a property).
+    const response = await expandRequest(app, [], { 'x-too-costly-threshold': '50' });
+
+    expect(response.status).toBe(422);
+    expect(response.body.resourceType).toBe('OperationOutcome');
+    expect(response.body.issue[0].code).toBe('too-costly');
+  });
+});

tests/tx/fixtures/test-library-snomed.yaml

@@ -0,0 +1,9 @@
+# Test library for whole-code-system SNOMED expansion tests - loads the
+# committed SNOMED test subset (copied into the terminology cache by the test's
+# beforeAll) alongside the automatic R5 core loading.
+
+base:
+  url: https://storage.googleapis.com/tx-fhir-org
+
+sources:
+  - snomed:snomed-testing.cache