From 8eafa1fab75fc6fa1293ccceafbaaa094c224a00 Mon Sep 17 00:00:00 2001 From: muratkeremozcan Date: Wed, 9 Sep 2026 04:22:06 -0500 Subject: [PATCH 1/4] fix(preflight): scope seeded-fault checks by the request each leg issues The clean-leg set of a `seeded-faults-scoped` check excluded the fault leg by identity alone. Another leg of the same operation, most often a sensitivity leg, can carry a request equal to the fault leg's. The environment answers both the same way, the manifestation witness fires on the clean leg, and the check reported a scoping violation for one observation counted twice. A leg whose built request equals the fault leg's is now dropped from the set. The comparison runs over the whole `ProbeRequest` in RFC 8785 form with `probeId` neutralised, so a field added to the request later joins the comparison with no edit at the call site, and key order never reads as a difference. Bounded to an operation AD-19 marks as changing no state, where one request has one answer for the length of the run. A mutating operation can answer the same request differently at two points in the sequence, so both legs stay in the set there. Fixture 59 now patches `list-b`, whose request differs from the fault leg's, and still fails. Fixtures 126, 127 and 128 cover the identical-request case at plan and reduce level and the mutating bound. The second fixture numbered 124 in `tests/preflight/reduce.test.ts` is renumbered 125. Claude-Session: https://claude.ai/code/session_01P4Adb9SkVtVbKCWPiV1Uwc --- CHANGELOG.md | 12 ++++++ src/core/preflight/plan.ts | 70 ++++++++++++++++++++++++++-------- tests/preflight/plan.test.ts | 40 ++++++++++++++++++- tests/preflight/reduce.test.ts | 22 ++++++++++- 4 files changed, 126 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a5fec81..d8f5236 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,18 @@ body. ## [Unreleased] +### Fixed + +- Pre-flight's `seeded-faults-scoped` check no longer fails on a clean leg that carries the fault + leg's own request. The clean-leg set excluded the fault leg by identity alone, so a sensitivity + leg spelling the same inputs got the same answer from the environment, the manifestation witness + fired on it, and the check reported a scoping violation for one observation counted twice. A leg + whose built request equals the fault leg's is now dropped from the set, compared over the whole + `ProbeRequest` in RFC 8785 form with the correlation id neutralised. The exclusion is bounded to + an operation AD-19 marks as changing no state, where one request has one answer for the length of + the run; on a mutating operation both legs stay in the set, since the same request issued twice + is two events the system may answer differently. + ## [1.3.0] - 2026-09-09 ### Fixed diff --git a/src/core/preflight/plan.ts b/src/core/preflight/plan.ts index 14c6e56..b0e4ce1 100644 --- a/src/core/preflight/plan.ts +++ b/src/core/preflight/plan.ts @@ -8,6 +8,7 @@ * the operation and the witness themselves: there is nothing to look an * identifier up in at reduce time. */ +import { serialize } from '../canonical/canonicalize.ts' import { checkInputsAgainstShape, isApiWitnessInputs, @@ -151,6 +152,21 @@ const requestOf = ( } } +/** + * A leg's request in the RFC 8785 form, with the correlation identifier + * neutralised because it is the leg id and differs between any two legs. + * + * Two legs whose signatures match ask the environment the same question, so the + * environment answers them the same way. Taken over the whole request, so a + * field added to `ProbeRequest` later is compared with no edit here. Canonical, + * because key order is a serialisation detail: two spellings of one request are + * one request. + */ +const requestSignature = ( + request: ProbeRequest, + artifactPath: string, +): string => serialize({ ...request, probeId: '' }, artifactPath) + /** where a leg came from, so a duplicate identifier names its own source. */ type LegOrigin = { readonly leg: PlannedLeg; readonly artifactPath: string } @@ -322,6 +338,9 @@ export const planPreflight: PlanStage = ( const legIdsByOperation = new Map() const scopeKey = (interfaceId: string, operationId: string): string => `${interfaceId}\u0000${operationId}` + // Every planned leg's request signature, so the seeded-fault branch below can + // ask whether a leg already in the group carries the fault leg's own request. + const signatureByLegId = new Map() const addLeg = ( legId: string, purpose: PlannedLegPurpose, @@ -329,21 +348,21 @@ export const planPreflight: PlanStage = ( operation: AnyOperation, inputs: WitnessInputs, artifactPath: string, - ): void => { - origins.push({ - leg: { - legId, - purpose, - request: requestOf(legId, interfaceId, operation, inputs), - operation, - inputs, - }, - artifactPath, - }) + ): PlannedLeg => { + const leg: PlannedLeg = { + legId, + purpose, + request: requestOf(legId, interfaceId, operation, inputs), + operation, + inputs, + } + origins.push({ leg, artifactPath }) + signatureByLegId.set(legId, requestSignature(leg.request, artifactPath)) const key = scopeKey(interfaceId, operation.operationId) const group = legIdsByOperation.get(key) if (group === undefined) legIdsByOperation.set(key, [legId]) else group.push(legId) + return leg } // 1. the sensitivity legs and their checks @@ -474,14 +493,14 @@ export const planPreflight: PlanStage = ( `the manifestation witness of ${defect.defectId}`, `${path}.inputs`, ) - // Read before the fault leg joins the group, which keeps the fault leg out - // of its own clean-leg set. - const cleanLegIds = [ + // The group is read before the fault leg joins it, which keeps the fault + // leg out of its own clean-leg set. + const group = [ ...(legIdsByOperation.get( scopeKey(witness.interfaceId, operation.operationId), ) ?? []), ] - addLeg( + const faultLeg = addLeg( witness.legId, 'seeded-fault', witness.interfaceId, @@ -489,6 +508,27 @@ export const planPreflight: PlanStage = ( witness.inputs, `${path}.legId`, ) + // A clean leg has to ask the environment a different question. Another + // leg of the same operation, most often a sensitivity leg, can carry a + // request equal to the fault leg's; the environment answers both the same + // way, so a witness firing there is the fault's own manifestation observed + // a second time. Such a leg is dropped, since nothing downstream can tell + // the two observations apart. + // + // Bounded to an operation AD-19 marks as changing no state, where one + // request has one answer for the length of the run, which is the property + // `state-reset` asserts over the same fixture. A mutating operation can + // answer the same request differently at two points in the sequence, so + // there the two legs are two events and the second answer is evidence of + // its own; both legs stay in the set. + const faultSignature = operation.stateChangeMarker + ? null + : requestSignature(faultLeg.request, `${path}.legId`) + const cleanLegIds = group.filter( + (legId) => + faultSignature === null || + signatureByLegId.get(legId) !== faultSignature, + ) checks.push({ kind: 'seeded-faults-scoped', defectId: defect.defectId, diff --git a/tests/preflight/plan.test.ts b/tests/preflight/plan.test.ts index 3a863bd..a94065c 100644 --- a/tests/preflight/plan.test.ts +++ b/tests/preflight/plan.test.ts @@ -330,8 +330,46 @@ describe('the plan as a whole', () => { const scoped = plan.checks.find( (check) => check.kind === 'seeded-faults-scoped', ) + // `list-a` is absent for the separate reason fixture 126 pins: it carries + // the fault leg's own request. What this fixture pins is that no + // `other-api` leg reached the set. expect( scoped?.kind === 'seeded-faults-scoped' ? scoped.cleanLegIds : [], - ).toEqual(['list-a', 'list-b']) + ).toEqual(['list-b']) + }) + + // The fixture's fault leg reads `list-things` with `limit: 1`, which is the + // sensitivity leg `list-a`'s request exactly. One request gets one answer, so + // a witness firing on `list-a` is the fault's own manifestation read a second + // time. `list-b` asks for `limit: 2` and stays. + it("126. drops a clean leg carrying the fault leg's own request, and keeps one that differs", () => { + const scoped = planOf().checks.find( + (check) => check.kind === 'seeded-faults-scoped', + ) + if (scoped?.kind !== 'seeded-faults-scoped') + throw new Error('the fixture declares one seeded defect') + expect(scoped.cleanLegIds).toEqual(['list-b']) + const requestOfLeg = (legId: string) => + planOf().legs.find((leg) => leg.legId === legId)?.request.channels + expect(requestOfLeg('list-a')).toEqual(requestOfLeg('fault-leg')) + expect(requestOfLeg('list-b')).not.toEqual(requestOfLeg('fault-leg')) + }) + + // The exclusion holds where one request has one answer. `create-thing` + // declares `stateChangeMarker: true`, so the same body posted twice is two + // events the environment may answer differently, and both legs stay. + it("128. keeps a clean leg carrying the fault leg's request when the operation changes state", () => { + const draft = probeDraft() + draft.defects[0].manifestationWitness.operationId = 'create-thing' + draft.defects[0].manifestationWitness.inputs = inputsOf({ + body: { kind: 'json', value: { name: 'alpha' } }, + }) + const plan = planOf(preflightContract, [ProbeSchema.parse(draft)]) + const scoped = plan.checks.find( + (check) => check.kind === 'seeded-faults-scoped', + ) + if (scoped?.kind !== 'seeded-faults-scoped') + throw new Error('the draft declares one seeded defect') + expect(scoped.cleanLegIds).toEqual(['create-a', 'create-b']) }) }) diff --git a/tests/preflight/reduce.test.ts b/tests/preflight/reduce.test.ts index de3ae81..bed501c 100644 --- a/tests/preflight/reduce.test.ts +++ b/tests/preflight/reduce.test.ts @@ -298,16 +298,34 @@ describe('the two seeded-fault checks, which are disjoint by construction', () = ) }) + // `list-b` reads `list-things` with `limit: 2` and the fault leg reads it + // with `limit: 1`, so this is a second question answered the way the fault + // leg's was: the defect shows outside its own leg. it('59. seeded-faults-scoped fails when the witness resolves true on a clean leg', () => { expect( outcomeOf( - { patches: { 'list-a': jsonPatch({ items: [{ broken: true }] }) } }, + { patches: { 'list-b': jsonPatch({ items: [{ broken: true }] }) } }, 'seeded-faults-scoped', 'list-things', ), ).toBe('failed') }) + // `list-a` carries the fault leg's request byte for byte, so an environment + // that answered one that way answered the other the same way. The plan drops + // such a leg (fixture 126) and the check stays satisfied; without that, every + // contract whose sensitivity legs cover the witness's own inputs failed + // pre-flight on one observation counted twice. + it("127. seeded-faults-scoped stays satisfied when the leg the witness fires on carries the fault leg's own request", () => { + expect( + outcomeOf( + { patches: { 'list-a': jsonPatch({ items: [{ broken: true }] }) } }, + 'seeded-faults-scoped', + 'list-things', + ), + ).toBe('satisfied') + }) + it('60. seeded-faults-scoped stays satisfied even when the witness resolves false on its own fault leg', () => { expect( outcomeOf( @@ -466,7 +484,7 @@ describe('the verdict itself', () => { // and it reports a mismatch as a failed `interface-present` verdict rather // than as a fault, which fixture 40 asserts. Throwing on them here would turn // a shipped verdict into a fault, so the fault reads `kind` and nothing else. - it('124. leaves an operation mismatch to the verdict that already reports it', () => { + it('125. leaves an operation mismatch to the verdict that already reports it', () => { expect( outcomeOf( { patches: { 'read-b': { operationId: 'create-thing' } } }, From dc7af54e0dc1a7d23e549467a603c8ea673ba842 Mon Sep 17 00:00:00 2001 From: muratkeremozcan Date: Wed, 9 Sep 2026 04:34:12 -0500 Subject: [PATCH 2/4] docs: record the request-based clean-leg rule in the learning path Step 20's rules list said what the two seeded-fault checks read and left "clean leg" as every other leg of the operation. A reader who assumes the set is built from leg identity alone misreads both the shipped behaviour and a mutating operation's result, so the rule and its `stateChangeMarker` bound are stated there. AC 11's `seeded-faults-scoped` row in story 6.2 carries an amendment note in the ADR-003 shape: the original row stays readable and the note says what superseded it. Step 27 is untouched. It covers AD-40's signature, qualification, and the witness match, and states no rule about pre-flight's clean-leg set. Claude-Session: https://claude.ai/code/session_01P4Adb9SkVtVbKCWPiV1Uwc --- ...-pre-flight-as-plan-observation-and-pure-verdict.md | 10 ++++++++++ .../project-knowledge/learning-path-step-by-step.md | 5 +++++ 2 files changed, 15 insertions(+) diff --git a/_bmad-output/implementation-artifacts/6-2-pre-flight-as-plan-observation-and-pure-verdict.md b/_bmad-output/implementation-artifacts/6-2-pre-flight-as-plan-observation-and-pure-verdict.md index d35a021..a82bbbf 100644 --- a/_bmad-output/implementation-artifacts/6-2-pre-flight-as-plan-observation-and-pure-verdict.md +++ b/_bmad-output/implementation-artifacts/6-2-pre-flight-as-plan-observation-and-pure-verdict.md @@ -570,6 +570,16 @@ Story 6.4's. | `seeded-faults-scoped` | the defect's witness resolves non-`true` on every clean leg of its operation | it resolves `true` on any clean leg | never | | `seeded-fault-fired` | the witness resolves `true` on its own fault leg | the witness is `null`, its leg has no observation, or the relation resolves `false` or `insufficient-evidence` | never | +> **Amended 2026-09-09.** The `seeded-faults-scoped` row says "every clean leg of its operation", +> and the shipped plan read "clean" as every other leg of that operation by leg id. A leg can carry +> a request identical to the fault leg's, most often a sensitivity leg spelling the witness's own +> inputs. The environment answers both the same way, so the check failed on one observation counted +> twice. A leg whose built request equals the fault leg's is now excluded from `cleanLegIds`, +> compared over the whole `ProbeRequest` in RFC 8785 form with `probeId` neutralised. The exclusion +> is bounded to an operation whose `stateChangeMarker` is false, where one request has one answer +> for the length of the run; a mutating operation keeps both legs, since the same request issued +> twice is two events the system may answer differently. Fixtures 126, 127, and 128. + **Anomalous** means `status >= 400`. The word is already the repository's: Story 6.1's conformance suite asserts `probe/observe-anomalous-status`, and `ProbeObservation.status` is already bounded to 100–599 at the port, so nothing new is assumed about the protocol here. diff --git a/_bmad-output/project-knowledge/learning-path-step-by-step.md b/_bmad-output/project-knowledge/learning-path-step-by-step.md index 0407272..2739e44 100644 --- a/_bmad-output/project-knowledge/learning-path-step-by-step.md +++ b/_bmad-output/project-knowledge/learning-path-step-by-step.md @@ -1794,6 +1794,11 @@ flowchart TD - `clean-control` reads only the control legs. AD-10's own example is two 404s from a good fixture. - The two seeded-fault checks are disjoint: one reads only clean legs, the other only the fault leg. Fold them together and one answer maps to no outcome the schema can spell. +- A clean leg is one that asks a different question. A leg carrying the fault leg's own request is + dropped from the set: one request gets one answer, so the witness firing there is the fault's own + manifestation read a second time. That holds for an operation whose `stateChangeMarker` is false. + A mutating operation can answer the same request differently at two points in the sequence, so + both legs stay in the set there. **Watch out:** From 11c3a76483314a5ebcd399d54b381cf92386d6f9 Mon Sep 17 00:00:00 2001 From: muratkeremozcan Date: Wed, 9 Sep 2026 05:00:34 -0500 Subject: [PATCH 3/4] fix(preflight): fail a seeded-fault scoping check that examined no clean leg An empty `cleanLegIds` resolved `satisfied`, so a defect whose operation carried no other leg certified its own scoping from no observation. That is the vacuity the file rejects everywhere else: `interface-present` is emitted only for an operation with a leg, and a sensitivity relation resolving `insufficient-evidence` fails. The check now fails on an empty set and the note names the cause, since an operation with no other leg and an operation whose every leg carries the fault leg's request are different authoring mistakes. The check carries `droppedLegIds` so the reducer can tell them apart. `docs/reference/glossary.md` documented the old answer and is updated. The request-equality filter widened the reach of that hole. AD-10's exemption case reaches it exactly: one keyless safe read, whose two control-observe legs both send the empty inputs the operation admits, and a defect seeded there matches every leg its operation has. `serialize` is not total over `JsonValue`. An integer outside the safe range, a lone surrogate, or nesting past AD-36's depth raises `non-canonicalizable-value`, and a 64-bit id in a query parameter is ordinary. Signing every leg eagerly turned that into a thrown fault from a stage that had only ever raised `StructuralFailure`. Signatures are now taken only for the legs of an operation carrying a seeded defect, and a request that cannot be canonicalised has no signature, matches nothing, and keeps its leg in the clean-leg set. The justification for the `stateChangeMarker` bound claimed `state-reset` established that one request has one answer for the length of the run. It does not: it compares the two control-observe legs alone, on one operation, and `runPreflight` issues legs in plan order, which puts mutating legs between reads of a different operation. The bound is caution, its cost is live, and the comment, the changelog, the story, and the learning path all say so now. Fixtures 130 and 131 cover the plan side, 132 and 133 the reducer. The second fixture numbered 123 in `tests/preflight/reduce.test.ts` is renumbered 129. Claude-Session: https://claude.ai/code/session_01P4Adb9SkVtVbKCWPiV1Uwc --- CHANGELOG.md | 21 +++-- ...ht-as-plan-observation-and-pure-verdict.md | 34 ++++++-- .../learning-path-step-by-step.md | 12 ++- docs/reference/glossary.md | 2 +- src/core/preflight/plan.ts | 85 ++++++++++++------- src/core/preflight/reduce.ts | 21 ++++- tests/preflight/fixtures/observations.ts | 59 +++++++++++++ tests/preflight/plan.test.ts | 57 +++++++++++++ tests/preflight/reduce.test.ts | 28 +++++- 9 files changed, 265 insertions(+), 54 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8f5236..d96daf0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,13 +14,20 @@ body. - Pre-flight's `seeded-faults-scoped` check no longer fails on a clean leg that carries the fault leg's own request. The clean-leg set excluded the fault leg by identity alone, so a sensitivity - leg spelling the same inputs got the same answer from the environment, the manifestation witness - fired on it, and the check reported a scoping violation for one observation counted twice. A leg - whose built request equals the fault leg's is now dropped from the set, compared over the whole - `ProbeRequest` in RFC 8785 form with the correlation id neutralised. The exclusion is bounded to - an operation AD-19 marks as changing no state, where one request has one answer for the length of - the run; on a mutating operation both legs stay in the set, since the same request issued twice - is two events the system may answer differently. + leg spelling the same inputs issued one request under two labels, the manifestation witness fired + on both, and the check reported a scoping violation over a leg the plan had no way to tell from + the fault leg. A leg whose built request equals the fault leg's is now dropped from the set, + compared over the whole `ProbeRequest` in RFC 8785 form with the correlation id neutralised. The + exclusion is bounded to an operation AD-19 marks as changing no state, and that bound is + caution: nothing available at plan time establishes that a system answers two identical mutating + requests differently, only that it may. The cost is live. A defect seeded on a mutating operation + whose manifestation witness repeats a sensitivity leg's inputs still fails this check. +- `seeded-faults-scoped` fails when its clean-leg set is empty. It was satisfied before, so a + defect seeded against an operation with no other leg certified its own scoping from no + observation at all. An empty set examined nothing and establishes nothing, which is the rule a + sensitivity relation resolving `insufficient-evidence` already follows. The note names the cause, + since the operation having no other leg and every other leg carrying the fault leg's request are + different authoring mistakes. ## [1.3.0] - 2026-09-09 diff --git a/_bmad-output/implementation-artifacts/6-2-pre-flight-as-plan-observation-and-pure-verdict.md b/_bmad-output/implementation-artifacts/6-2-pre-flight-as-plan-observation-and-pure-verdict.md index a82bbbf..49a8eaa 100644 --- a/_bmad-output/implementation-artifacts/6-2-pre-flight-as-plan-observation-and-pure-verdict.md +++ b/_bmad-output/implementation-artifacts/6-2-pre-flight-as-plan-observation-and-pure-verdict.md @@ -570,15 +570,31 @@ Story 6.4's. | `seeded-faults-scoped` | the defect's witness resolves non-`true` on every clean leg of its operation | it resolves `true` on any clean leg | never | | `seeded-fault-fired` | the witness resolves `true` on its own fault leg | the witness is `null`, its leg has no observation, or the relation resolves `false` or `insufficient-evidence` | never | -> **Amended 2026-09-09.** The `seeded-faults-scoped` row says "every clean leg of its operation", -> and the shipped plan read "clean" as every other leg of that operation by leg id. A leg can carry -> a request identical to the fault leg's, most often a sensitivity leg spelling the witness's own -> inputs. The environment answers both the same way, so the check failed on one observation counted -> twice. A leg whose built request equals the fault leg's is now excluded from `cleanLegIds`, -> compared over the whole `ProbeRequest` in RFC 8785 form with `probeId` neutralised. The exclusion -> is bounded to an operation whose `stateChangeMarker` is false, where one request has one answer -> for the length of the run; a mutating operation keeps both legs, since the same request issued -> twice is two events the system may answer differently. Fixtures 126, 127, and 128. +> +> **Amended 2026-09-09, two rules.** +> +> 1. The row says "every clean leg of its operation", and the shipped plan read "clean" as every +> other leg of that operation by leg id. A leg can carry a request identical to the fault leg's, +> most often a sensitivity leg spelling the witness's own inputs, and the plan has nothing that +> tells the two apart: same operation, same inputs, same request bytes. The check failed over a +> leg it could not distinguish from the fault leg. A leg whose built request equals the fault +> leg's is now excluded from `cleanLegIds`, compared over the whole `ProbeRequest` in RFC 8785 +> form with `probeId` neutralised. The exclusion is bounded to an operation whose +> `stateChangeMarker` is false. That bound is conservatism: nothing at plan time establishes that +> a mutating operation answers two identical requests differently, only that it may, since a +> request that changes state is a different event the second time it is issued. The cost is +> live and named: a defect seeded on `create-thing` whose witness posts `{name: 'alpha'}`, the +> same body sensitivity leg `create-a` sends, still reports `the manifestation witness fires on +> clean leg "create-a"`. Comparing the two legs' observations at reduce time would close it, and +> that comparison is left open pending a decision. Fixtures 126, 127, and 128. +> 2. An empty `cleanLegIds` resolved `satisfied`, which certified scoping from no observation. It +> fails now, on the rule the `input-sensitivity` row already runs on: a check that examined +> nothing has established nothing. Both causes are reachable. An operation whose only leg is the +> fault leg reached it before this change, and rule 1 adds the operation every one of whose legs +> carries the fault leg's request, which is AD-10's exemption case exactly: one keyless safe read +> whose two control-observe legs both send the empty inputs the operation admits. The check +> carries `droppedLegIds` so the note can say which cause it was. Fixtures 130, 131, 132, and +> 133. **Anomalous** means `status >= 400`. The word is already the repository's: Story 6.1's conformance suite asserts `probe/observe-anomalous-status`, and `ProbeObservation.status` is already bounded to diff --git a/_bmad-output/project-knowledge/learning-path-step-by-step.md b/_bmad-output/project-knowledge/learning-path-step-by-step.md index 2739e44..16a72b5 100644 --- a/_bmad-output/project-knowledge/learning-path-step-by-step.md +++ b/_bmad-output/project-knowledge/learning-path-step-by-step.md @@ -1795,10 +1795,14 @@ flowchart TD - The two seeded-fault checks are disjoint: one reads only clean legs, the other only the fault leg. Fold them together and one answer maps to no outcome the schema can spell. - A clean leg is one that asks a different question. A leg carrying the fault leg's own request is - dropped from the set: one request gets one answer, so the witness firing there is the fault's own - manifestation read a second time. That holds for an operation whose `stateChangeMarker` is false. - A mutating operation can answer the same request differently at two points in the sequence, so - both legs stay in the set there. + dropped from the set, because the plan has nothing that tells the two apart. The drop is bounded + to an operation whose `stateChangeMarker` is false, which is caution: nothing establishes that a + mutating operation answers two identical requests differently, only that it may. A defect seeded + on a mutating operation whose witness repeats a sensitivity leg's inputs still fails this check. +- An empty clean-leg set fails. The check examined nothing, and a check that examined nothing has + established nothing, which is the same rule `insufficient-evidence` gets. The note says which + cause emptied it: the operation had no other leg, or every other leg carried the fault leg's own + request. **Watch out:** diff --git a/docs/reference/glossary.md b/docs/reference/glossary.md index 8367ff7..2e54109 100644 --- a/docs/reference/glossary.md +++ b/docs/reference/glossary.md @@ -79,7 +79,7 @@ The six kinds a `PreflightVerdict` can carry. | `state-reset` | Does the declared fixture reset return the system to a known state? | | `clean-control` | Does the leg that should show nothing wrong in fact show nothing wrong? | | `seeded-fault-fired` | Did the seeded fault manifest where it was supposed to? | -| `seeded-faults-scoped` | Did a manifestation witness fire on a clean leg, where it should not have? A leg with no observation cannot fire one, so a missing clean leg leaves this `satisfied`. | +| `seeded-faults-scoped` | Did a manifestation witness fire on a clean leg, where it should not have? A clean leg is one that asks a different question, so a leg sending the fault leg's own request is left out. With no clean leg left to ask, this fails and says why. | ## Contract vocabulary diff --git a/src/core/preflight/plan.ts b/src/core/preflight/plan.ts index b0e4ce1..7c03a1a 100644 --- a/src/core/preflight/plan.ts +++ b/src/core/preflight/plan.ts @@ -21,6 +21,7 @@ import { referenceSetKeysOf } from '../evaluate/evidence-resolution.ts' import type { ReferenceSetKeys } from '../evaluate/resolution.ts' import { StructuralFailure } from '../failure-codes.ts' import type { EvalContract } from '../schemas/eval-contract.ts' +import { RuntimeFault } from '../schemas/faults.ts' import type { AnyOperation, PermittedInterface } from '../schemas/interface.ts' import { operationsOf } from '../schemas/interface.ts' import type { ProbeRequest } from '../schemas/port-messages.ts' @@ -33,6 +34,7 @@ import type { WitnessInputs, } from '../schemas/sensitivity-witness.ts' import type { PlanStage } from '../stage-contracts.ts' +import { PREFLIGHT_ARTIFACT_PATH } from './projection.ts' import { referenceSetMembers } from './witness-evidence.ts' export type PreflightPlanInput = { @@ -78,6 +80,9 @@ export type PlannedCheck = readonly witness: ManifestationWitness readonly operation: AnyOperation readonly cleanLegIds: readonly string[] + // The legs dropped for carrying the fault leg's own request, so the + // reducer can say which of the two ways an empty clean-leg set arose. + readonly droppedLegIds: readonly string[] } | { readonly kind: 'seeded-fault-fired' @@ -155,17 +160,26 @@ const requestOf = ( /** * A leg's request in the RFC 8785 form, with the correlation identifier * neutralised because it is the leg id and differs between any two legs. + * Two legs whose signatures match issue one request under two labels. Taken over + * the whole request, so a field added to `ProbeRequest` later is compared with + * no edit here, and canonical, because two key orders spell one request. * - * Two legs whose signatures match ask the environment the same question, so the - * environment answers them the same way. Taken over the whole request, so a - * field added to `ProbeRequest` later is compared with no edit here. Canonical, - * because key order is a serialisation detail: two spellings of one request are - * one request. + * `null` when the request holds a value RFC 8785 cannot serialise. `JsonValue` + * admits an integer outside the safe range, a lone surrogate, and nesting past + * AD-36's depth, and `serialize` raises `non-canonicalizable-value` on each. + * Planning stays a plan: `planPreflight` reports declaration defects as + * `StructuralFailure`, and pre-flight ships a verdict wherever it can. A request + * nothing can canonicalise is a request nothing can prove identical, and `null` + * matches no other signature, so such a leg stays in the clean-leg set. */ -const requestSignature = ( - request: ProbeRequest, - artifactPath: string, -): string => serialize({ ...request, probeId: '' }, artifactPath) +const requestSignature = (request: ProbeRequest): string | null => { + try { + return serialize({ ...request, probeId: '' }, PREFLIGHT_ARTIFACT_PATH) + } catch (error) { + if (error instanceof RuntimeFault) return null + throw error + } +} /** where a leg came from, so a duplicate identifier names its own source. */ type LegOrigin = { readonly leg: PlannedLeg; readonly artifactPath: string } @@ -338,9 +352,11 @@ export const planPreflight: PlanStage = ( const legIdsByOperation = new Map() const scopeKey = (interfaceId: string, operationId: string): string => `${interfaceId}\u0000${operationId}` - // Every planned leg's request signature, so the seeded-fault branch below can - // ask whether a leg already in the group carries the fault leg's own request. - const signatureByLegId = new Map() + // Every planned leg's built request, so the seeded-fault branch below can ask + // whether a leg already in the group carries the fault leg's own. Signatures + // are taken at that site, so a contract seeding no defect canonicalises + // nothing. + const requestByLegId = new Map() const addLeg = ( legId: string, purpose: PlannedLegPurpose, @@ -357,7 +373,7 @@ export const planPreflight: PlanStage = ( inputs, } origins.push({ leg, artifactPath }) - signatureByLegId.set(legId, requestSignature(leg.request, artifactPath)) + requestByLegId.set(legId, leg.request) const key = scopeKey(interfaceId, operation.operationId) const group = legIdsByOperation.get(key) if (group === undefined) legIdsByOperation.set(key, [legId]) @@ -508,33 +524,40 @@ export const planPreflight: PlanStage = ( witness.inputs, `${path}.legId`, ) - // A clean leg has to ask the environment a different question. Another - // leg of the same operation, most often a sensitivity leg, can carry a - // request equal to the fault leg's; the environment answers both the same - // way, so a witness firing there is the fault's own manifestation observed - // a second time. Such a leg is dropped, since nothing downstream can tell - // the two observations apart. + // A clean leg has to ask a different question. Another leg of the same + // operation, most often a sensitivity leg, can carry a request equal to + // the fault leg's, and the plan has nothing that tells the two apart: same + // operation, same inputs, same request bytes. A witness firing on such a + // leg says nothing about scope, so it is dropped and the reducer is told + // it was. // - // Bounded to an operation AD-19 marks as changing no state, where one - // request has one answer for the length of the run, which is the property - // `state-reset` asserts over the same fixture. A mutating operation can - // answer the same request differently at two points in the sequence, so - // there the two legs are two events and the second answer is evidence of - // its own; both legs stay in the set. + // Bounded to an operation AD-19 marks as changing no state. That bound is + // caution: nothing here establishes that a mutating operation answers two + // identical requests differently, only that it may, since a request that + // changes state is a different event the second time it is issued. The cost is named in the changelog: a defect seeded on a + // mutating operation whose witness repeats a sensitivity leg's inputs + // still fails this check. const faultSignature = operation.stateChangeMarker ? null - : requestSignature(faultLeg.request, `${path}.legId`) - const cleanLegIds = group.filter( - (legId) => - faultSignature === null || - signatureByLegId.get(legId) !== faultSignature, - ) + : requestSignature(faultLeg.request) + const dropped = + faultSignature === null + ? [] + : group.filter((legId) => { + const request = requestByLegId.get(legId) + return ( + request !== undefined && + requestSignature(request) === faultSignature + ) + }) + const cleanLegIds = group.filter((legId) => !dropped.includes(legId)) checks.push({ kind: 'seeded-faults-scoped', defectId: defect.defectId, witness, operation, cleanLegIds, + droppedLegIds: dropped, }) checks.push({ kind: 'seeded-fault-fired', diff --git a/src/core/preflight/reduce.ts b/src/core/preflight/reduce.ts index b46d276..5a68035 100644 --- a/src/core/preflight/reduce.ts +++ b/src/core/preflight/reduce.ts @@ -306,7 +306,26 @@ export const reducePreflight: ReduceStage< return check(planned.kind, null, 'satisfied', null) } case 'seeded-faults-scoped': { - const { witness, defectId } = planned + const { witness, defectId, droppedLegIds } = planned + // A check over no clean leg examined nothing, and a check that + // examined nothing has established nothing, which is the rule the + // `input-sensitivity` row above already runs on. Satisfied here would + // certify scoping from zero evidence, and it would do so on the two + // contracts least able to afford it: one whose defect names the only + // leg its operation has, and one whose every other leg repeats the + // fault leg's request. Those are different authoring mistakes, so the + // note says which one this was. + if (planned.cleanLegIds.length === 0) { + const named = droppedLegIds.map((legId) => `"${legId}"`).join(', ') + return check( + planned.kind, + witness.operationId, + 'failed', + droppedLegIds.length === 0 + ? `${defectId}: the operation has no leg besides the fault leg, so nothing here establishes that the defect is scoped to it` + : `${defectId}: every other leg of the operation carries the fault leg's own request (${named}), so nothing here establishes that the defect is scoped to it`, + ) + } for (const legId of planned.cleanLegIds) { const resolved = resolveAgainst( witness, diff --git a/tests/preflight/fixtures/observations.ts b/tests/preflight/fixtures/observations.ts index 668566e..de2277f 100644 --- a/tests/preflight/fixtures/observations.ts +++ b/tests/preflight/fixtures/observations.ts @@ -478,6 +478,65 @@ export const cleanControlProbe: Probe = Probe.parse({ /** a deep copy of the seeded probe a test may mutate before parsing it back. */ export const probeDraft = (): any => structuredClone(seededProbe) +/** + * AD-10's exemption case as a whole contract: one safe read declaring no key in + * any channel. Its only legs are the two control-observe legs, and both carry + * the empty inputs the operation admits, so a defect seeded there has a + * manifestation witness that matches every leg of its own operation. + */ +export const keylessReadContract: EvalContract = EvalContract.parse({ + ...contractLiteral, + contractId: 'preflight-fixture-keyless', + permittedInterfaces: [ + { + logicalId: 'thing-api', + kind: 'api', + operations: [ + { + operationId: 'read-health', + method: 'GET', + pathTemplate: '/health', + stateChangeMarker: false, + requestShape: { + path: emptyChannel(), + query: emptyChannel(), + header: emptyChannel(), + body: emptyChannel(), + }, + responseDescriptor: { + requiredKeys: [], + permittedKeys: ['ok'], + types: { ok: 'boolean' }, + successIndicator: null, + channelRoles: null, + collectionLocations: null, + }, + volatilePointers: [], + sensitivityWitness: null, + }, + ], + }, + ], +}) + +/** the seeded probe aimed at that contract's only operation. */ +export const keylessDefectProbe: Probe = (() => { + const draft = probeDraft() + draft.probeId = 'P-003' + draft.defects[0].manifestationWitness.operationId = 'read-health' + draft.defects[0].manifestationWitness.inputs = inputsOf() + return Probe.parse(draft) +})() + +/** the same probe aimed at an operation the plan gives no other leg. */ +export const lonelyDefectProbe: Probe = (() => { + const draft = probeDraft() + draft.probeId = 'P-004' + draft.defects[0].manifestationWitness.operationId = 'reset-things' + draft.defects[0].manifestationWitness.inputs = inputsOf() + return Probe.parse(draft) +})() + export type ObservationPatch = { readonly status?: number readonly body?: ProbeObservedBody diff --git a/tests/preflight/plan.test.ts b/tests/preflight/plan.test.ts index a94065c..c648865 100644 --- a/tests/preflight/plan.test.ts +++ b/tests/preflight/plan.test.ts @@ -17,6 +17,8 @@ import { cleanControlProbe, contractDraft, inputsOf, + keylessDefectProbe, + keylessReadContract, parseContract, preflightContract, probeDraft, @@ -372,4 +374,59 @@ describe('the plan as a whole', () => { throw new Error('the draft declares one seeded defect') expect(scoped.cleanLegIds).toEqual(['create-a', 'create-b']) }) + + // AD-10's exemption case: one keyless safe read, whose only legs are the two + // control-observe legs, both carrying the empty inputs the operation admits. + // A defect seeded there matches every leg it has. The reducer is told which + // legs went, because an empty set it read as satisfied would certify scoping + // from nothing (fixture 132). + it("130. names the legs it dropped when the fault leg's request matches every leg of its operation", () => { + const plan = planPreflight({ + contract: keylessReadContract, + probes: [keylessDefectProbe], + runId: 'run-1', + }) + const scoped = plan.checks.find( + (check) => check.kind === 'seeded-faults-scoped', + ) + if (scoped?.kind !== 'seeded-faults-scoped') + throw new Error('the probe declares one seeded defect') + expect(scoped.cleanLegIds).toEqual([]) + expect(scoped.droppedLegIds).toEqual([ + 'preflight-control-observe', + 'preflight-control-observe-2', + ]) + }) + + // `JsonValue` admits an integer outside the safe range and RFC 8785 does not, + // so the signature of such a request is unavailable. Planning stays a plan: + // no fault is thrown, and a leg that cannot be proved identical stays in the + // clean-leg set. + it('131. keeps a leg whose request holds a value RFC 8785 cannot serialise', () => { + const draft = contractDraft() + const listThings = draft.permittedInterfaces[0].operations.find( + (operation: { operationId: string }) => + operation.operationId === 'list-things', + ) + listThings.sensitivityWitness.legs[0].inputs.query = { limit: 1e21 } + const cleanLegsOf = (probes: readonly Probe[]) => { + const scoped = planOf(parseContract(draft), probes).checks.find( + (check) => check.kind === 'seeded-faults-scoped', + ) + if (scoped?.kind !== 'seeded-faults-scoped') + throw new Error('the probe declares one seeded defect') + return scoped.cleanLegIds + } + // The unserialisable leg is the clean one here. + expect(cleanLegsOf([seededProbe])).toEqual(['list-a', 'list-b']) + // And here it is the fault leg, so no leg is dropped at all. + const faulty = probeDraft() + faulty.defects[0].manifestationWitness.inputs = inputsOf({ + query: { limit: 1e21 }, + }) + expect(cleanLegsOf([ProbeSchema.parse(faulty)])).toEqual([ + 'list-a', + 'list-b', + ]) + }) }) diff --git a/tests/preflight/reduce.test.ts b/tests/preflight/reduce.test.ts index bed501c..48bf364 100644 --- a/tests/preflight/reduce.test.ts +++ b/tests/preflight/reduce.test.ts @@ -15,6 +15,9 @@ import { cleanControlProbe, contractDraft, jsonBody, + keylessDefectProbe, + keylessReadContract, + lonelyDefectProbe, type ObservationPatch, observationsFor, parseContract, @@ -326,6 +329,29 @@ describe('the two seeded-fault checks, which are disjoint by construction', () = ).toBe('satisfied') }) + // An empty clean-leg set examined nothing, so it establishes nothing. The two + // ways it empties are different authoring mistakes and the note says which. + it("132. seeded-faults-scoped fails when every other leg of the operation carries the fault leg's request", () => { + const { checks } = verdictOf({ + contract: keylessReadContract, + probes: [keylessDefectProbe], + }) + const scoped = checkFor(checks, 'seeded-faults-scoped', 'read-health') + expect(scoped.outcome).toBe('failed') + expect(scoped.note).toContain("carries the fault leg's own request") + expect(scoped.note).toContain('"preflight-control-observe"') + }) + + it('133. seeded-faults-scoped fails when the operation has no leg besides the fault leg', () => { + const scoped = checkFor( + verdictOf({ probes: [lonelyDefectProbe] }).checks, + 'seeded-faults-scoped', + 'reset-things', + ) + expect(scoped.outcome).toBe('failed') + expect(scoped.note).toContain('no leg besides the fault leg') + }) + it('60. seeded-faults-scoped stays satisfied even when the witness resolves false on its own fault leg', () => { expect( outcomeOf( @@ -447,7 +473,7 @@ describe('the verdict itself', () => { // of a command contract with a schema-valid HTTP observation and pre-flight // would report four checks satisfied with no command ever run. `probeId` ties // the answer to the question; only `kind` says it answered the same question. - it('123. raises port-contract-violation when the observation answers the other mechanism', () => { + it('129. raises port-contract-violation when the observation answers the other mechanism', () => { const plan = planPreflight({ contract: preflightContract, probes: [seededProbe], From d37aef536e18fb8921becd6765520c86d810742b Mon Sep 17 00:00:00 2001 From: muratkeremozcan Date: Wed, 9 Sep 2026 05:13:09 -0500 Subject: [PATCH 4/4] fix(preflight): decide seeded-fault scoping on the answers the legs got The clean-leg drop moves from `planPreflight` to `reducePreflight`, where both legs' answers are in hand. A clean leg is dropped when it issued the fault leg's request and received the fault leg's answer, which makes it the fault leg's own probe under a second label. Both halves are required. Answers alone would drop AD-10's own worked example, two distinct nonexistent identifiers both returning 404, which are exactly the legs this check exists to read. Requests alone are what the plan could see, and identical requests can still be answered differently. The answer half compares the leg's evidence, which is everything a relation can address. It carries AD-11's projected body, so a field the operation declares volatile is already out of it and a server-minted identifier stops being a difference. That is what makes the same request to a mutating operation comparable, and it closes the case the plan-side version had to leave open: a defect seeded on `create-thing` whose witness posts a sensitivity leg's body now passes when the two legs were answered alike, and fails when they were not. Emptiness is tested on the set that survives the drop, since the drop is what can empty it. The note names the cause: the operation has no leg besides the fault leg, or every leg the plan named ran the fault leg's probe. `digestArtifact` raises `non-canonicalizable-value` on an integer outside the safe range or a lone surrogate, both of which `JsonValue` admits. It is caught and read as no digest, which matches nothing, so a pair that cannot be compared stays a pair the check reads and the stage still returns a verdict. `plan.ts` and its tests are back to what main carries. Fixtures 126, 127, 128, 130, 131, 132, and 133 all sit at reduce level now. Claude-Session: https://claude.ai/code/session_01P4Adb9SkVtVbKCWPiV1Uwc --- CHANGELOG.md | 32 +++-- ...ht-as-plan-observation-and-pure-verdict.md | 43 +++--- .../learning-path-step-by-step.md | 21 +-- docs/reference/glossary.md | 2 +- src/core/preflight/plan.ts | 93 ++----------- src/core/preflight/reduce.ts | 102 ++++++++++++-- tests/preflight/plan.test.ts | 97 +------------ tests/preflight/reduce.test.ts | 130 ++++++++++++++++-- 8 files changed, 277 insertions(+), 243 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5cb930b..3a09f8a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,22 +24,24 @@ body. ### Fixed -- Pre-flight's `seeded-faults-scoped` check no longer fails on a clean leg that carries the fault - leg's own request. The clean-leg set excluded the fault leg by identity alone, so a sensitivity - leg spelling the same inputs issued one request under two labels, the manifestation witness fired - on both, and the check reported a scoping violation over a leg the plan had no way to tell from - the fault leg. A leg whose built request equals the fault leg's is now dropped from the set, - compared over the whole `ProbeRequest` in RFC 8785 form with the correlation id neutralised. The - exclusion is bounded to an operation AD-19 marks as changing no state, and that bound is - caution: nothing available at plan time establishes that a system answers two identical mutating - requests differently, only that it may. The cost is live. A defect seeded on a mutating operation - whose manifestation witness repeats a sensitivity leg's inputs still fails this check. -- `seeded-faults-scoped` fails when its clean-leg set is empty. It was satisfied before, so a - defect seeded against an operation with no other leg certified its own scoping from no - observation at all. An empty set examined nothing and establishes nothing, which is the rule a +- Pre-flight's `seeded-faults-scoped` check no longer fails on a leg that is the fault leg's own + probe wearing a second label. The clean-leg set excluded the fault leg by leg id alone, so a + sensitivity witness leg spelling the manifestation witness's inputs was read as independent + evidence, the witness fired on it, and the check reported a scoping violation. A clean leg is now + dropped when it issued the fault leg's request and received the fault leg's answer. Both are + compared as canonical digests: the request with its correlation identifier neutralised, the + answer as the evidence a relation can address, which carries AD-11's projected body, so a field + the operation declares volatile is already out of it and a server-minted identifier stops being a + difference. Both halves are required. Answers alone would drop AD-10's own worked example of two + distinct nonexistent identifiers both returning 404, and requests alone cannot see that a system + answered one request two ways. The comparison lives in the reducer, where the answers are in + hand, beside the `state-reset` row that already compares two legs there. +- `seeded-faults-scoped` fails when no clean leg survives that comparison. It was satisfied before, + so a defect seeded against an operation with no other leg certified its own scoping from no + observation at all. A check that examined nothing has established nothing, which is the rule a sensitivity relation resolving `insufficient-evidence` already follows. The note names the cause, - since the operation having no other leg and every other leg carrying the fault leg's request are - different authoring mistakes. + since an operation with no other leg and an operation every one of whose legs ran the fault leg's + probe are different authoring mistakes. ## [1.3.0] - 2026-09-09 diff --git a/_bmad-output/implementation-artifacts/6-2-pre-flight-as-plan-observation-and-pure-verdict.md b/_bmad-output/implementation-artifacts/6-2-pre-flight-as-plan-observation-and-pure-verdict.md index 49a8eaa..ba90c4e 100644 --- a/_bmad-output/implementation-artifacts/6-2-pre-flight-as-plan-observation-and-pure-verdict.md +++ b/_bmad-output/implementation-artifacts/6-2-pre-flight-as-plan-observation-and-pure-verdict.md @@ -573,27 +573,30 @@ Story 6.4's. > > **Amended 2026-09-09, two rules.** > -> 1. The row says "every clean leg of its operation", and the shipped plan read "clean" as every -> other leg of that operation by leg id. A leg can carry a request identical to the fault leg's, -> most often a sensitivity leg spelling the witness's own inputs, and the plan has nothing that -> tells the two apart: same operation, same inputs, same request bytes. The check failed over a -> leg it could not distinguish from the fault leg. A leg whose built request equals the fault -> leg's is now excluded from `cleanLegIds`, compared over the whole `ProbeRequest` in RFC 8785 -> form with `probeId` neutralised. The exclusion is bounded to an operation whose -> `stateChangeMarker` is false. That bound is conservatism: nothing at plan time establishes that -> a mutating operation answers two identical requests differently, only that it may, since a -> request that changes state is a different event the second time it is issued. The cost is -> live and named: a defect seeded on `create-thing` whose witness posts `{name: 'alpha'}`, the -> same body sensitivity leg `create-a` sends, still reports `the manifestation witness fires on -> clean leg "create-a"`. Comparing the two legs' observations at reduce time would close it, and -> that comparison is left open pending a decision. Fixtures 126, 127, and 128. -> 2. An empty `cleanLegIds` resolved `satisfied`, which certified scoping from no observation. It +> 1. The row says "every clean leg of its operation", and the shipped reducer read "clean" as every +> other leg of that operation by leg id. A leg can run the fault leg's own probe under a second +> label, most often a sensitivity witness leg spelling the manifestation witness's inputs, and a +> witness firing there is the fault leg's own manifestation read a second time. The reducer now +> drops a clean leg that issued the fault leg's request and received the fault leg's answer. +> Both halves are required. Answers alone would drop AD-10's own worked example, two distinct +> nonexistent identifiers both returning 404, which are exactly the legs this check exists to +> read; requests alone are what the plan can see, and identical requests can still be answered +> differently. Both sides are canonical digests, the request with `probeId` neutralised and the +> answer as the leg's evidence with `observationId` neutralised. Evidence is the right side of +> the comparison because it is everything a relation can address, and it carries AD-11's +> projected body, so a field the operation declares volatile is already out of it. The comparison +> is in `reducePreflight` beside the `state-reset` row, which already compares two legs' +> projections there. An earlier revision of this change compared requests alone in +> `planPreflight` and narrowed the drop to `stateChangeMarker: false`, which left a defect seeded +> on `create-thing` failing the check whenever its witness posted a sensitivity leg's body. That +> case passes now: the answers decide it. Fixtures 126, 127, 128, 131, and 132. +> 2. An empty clean-leg set resolved `satisfied`, which certified scoping from no observation. It > fails now, on the rule the `input-sensitivity` row already runs on: a check that examined -> nothing has established nothing. Both causes are reachable. An operation whose only leg is the -> fault leg reached it before this change, and rule 1 adds the operation every one of whose legs -> carries the fault leg's request, which is AD-10's exemption case exactly: one keyless safe read -> whose two control-observe legs both send the empty inputs the operation admits. The check -> carries `droppedLegIds` so the note can say which cause it was. Fixtures 130, 131, 132, and +> nothing has established nothing. Emptiness is tested on the set that survives the drop, since +> the drop is what can empty it. Both causes are reachable and the note says which. An operation +> whose only leg is the fault leg reached it before this change. The drop adds AD-10's exemption +> case exactly: one keyless safe read whose two control-observe legs both send the empty inputs +> the operation admits, answered alike, so every leg the plan named is dropped. Fixtures 130 and > 133. **Anomalous** means `status >= 400`. The word is already the repository's: Story 6.1's conformance diff --git a/_bmad-output/project-knowledge/learning-path-step-by-step.md b/_bmad-output/project-knowledge/learning-path-step-by-step.md index 42a4570..0504b7f 100644 --- a/_bmad-output/project-knowledge/learning-path-step-by-step.md +++ b/_bmad-output/project-knowledge/learning-path-step-by-step.md @@ -1794,15 +1794,18 @@ flowchart TD - `clean-control` reads only the control legs. AD-10's own example is two 404s from a good fixture. - The two seeded-fault checks are disjoint: one reads only clean legs, the other only the fault leg. Fold them together and one answer maps to no outcome the schema can spell. -- A clean leg is one that asks a different question. A leg carrying the fault leg's own request is - dropped from the set, because the plan has nothing that tells the two apart. The drop is bounded - to an operation whose `stateChangeMarker` is false, which is caution: nothing establishes that a - mutating operation answers two identical requests differently, only that it may. A defect seeded - on a mutating operation whose witness repeats a sensitivity leg's inputs still fails this check. -- An empty clean-leg set fails. The check examined nothing, and a check that examined nothing has - established nothing, which is the same rule `insufficient-evidence` gets. The note says which - cause emptied it: the operation had no other leg, or every other leg carried the fault leg's own - request. +- A clean leg is one that asks a different question. The reducer drops a leg that issued the fault + leg's request and got the fault leg's answer, since that leg is the fault leg's own probe under a + second label. Both halves are needed: answers alone would drop two 404s from two distinct + nonexistent identifiers, which is AD-10's own example of a good fixture, and requests alone cannot + see that one request was answered two ways. +- The answer half compares the evidence, which is what a relation can address. It carries the + projected body, so a field the operation declares volatile is out of it already and a + server-minted id stops being a difference. +- An empty clean-leg set fails, and emptiness is tested on what survived the drop. The check + examined nothing, and a check that examined nothing has established nothing, which is the same + rule `insufficient-evidence` gets. The note says which cause emptied it: the operation had no + other leg, or every leg the plan named ran the fault leg's own probe. **Watch out:** diff --git a/docs/reference/glossary.md b/docs/reference/glossary.md index 2e54109..fd858d6 100644 --- a/docs/reference/glossary.md +++ b/docs/reference/glossary.md @@ -79,7 +79,7 @@ The six kinds a `PreflightVerdict` can carry. | `state-reset` | Does the declared fixture reset return the system to a known state? | | `clean-control` | Does the leg that should show nothing wrong in fact show nothing wrong? | | `seeded-fault-fired` | Did the seeded fault manifest where it was supposed to? | -| `seeded-faults-scoped` | Did a manifestation witness fire on a clean leg, where it should not have? A clean leg is one that asks a different question, so a leg sending the fault leg's own request is left out. With no clean leg left to ask, this fails and says why. | +| `seeded-faults-scoped` | Did a manifestation witness fire on a clean leg, where it should not have? A clean leg is one that asks a different question, so a leg that sent the fault leg's request and got back the fault leg's answer is left out. With no clean leg left to ask, this fails and says why. | ## Contract vocabulary diff --git a/src/core/preflight/plan.ts b/src/core/preflight/plan.ts index 7c03a1a..14c6e56 100644 --- a/src/core/preflight/plan.ts +++ b/src/core/preflight/plan.ts @@ -8,7 +8,6 @@ * the operation and the witness themselves: there is nothing to look an * identifier up in at reduce time. */ -import { serialize } from '../canonical/canonicalize.ts' import { checkInputsAgainstShape, isApiWitnessInputs, @@ -21,7 +20,6 @@ import { referenceSetKeysOf } from '../evaluate/evidence-resolution.ts' import type { ReferenceSetKeys } from '../evaluate/resolution.ts' import { StructuralFailure } from '../failure-codes.ts' import type { EvalContract } from '../schemas/eval-contract.ts' -import { RuntimeFault } from '../schemas/faults.ts' import type { AnyOperation, PermittedInterface } from '../schemas/interface.ts' import { operationsOf } from '../schemas/interface.ts' import type { ProbeRequest } from '../schemas/port-messages.ts' @@ -34,7 +32,6 @@ import type { WitnessInputs, } from '../schemas/sensitivity-witness.ts' import type { PlanStage } from '../stage-contracts.ts' -import { PREFLIGHT_ARTIFACT_PATH } from './projection.ts' import { referenceSetMembers } from './witness-evidence.ts' export type PreflightPlanInput = { @@ -80,9 +77,6 @@ export type PlannedCheck = readonly witness: ManifestationWitness readonly operation: AnyOperation readonly cleanLegIds: readonly string[] - // The legs dropped for carrying the fault leg's own request, so the - // reducer can say which of the two ways an empty clean-leg set arose. - readonly droppedLegIds: readonly string[] } | { readonly kind: 'seeded-fault-fired' @@ -157,30 +151,6 @@ const requestOf = ( } } -/** - * A leg's request in the RFC 8785 form, with the correlation identifier - * neutralised because it is the leg id and differs between any two legs. - * Two legs whose signatures match issue one request under two labels. Taken over - * the whole request, so a field added to `ProbeRequest` later is compared with - * no edit here, and canonical, because two key orders spell one request. - * - * `null` when the request holds a value RFC 8785 cannot serialise. `JsonValue` - * admits an integer outside the safe range, a lone surrogate, and nesting past - * AD-36's depth, and `serialize` raises `non-canonicalizable-value` on each. - * Planning stays a plan: `planPreflight` reports declaration defects as - * `StructuralFailure`, and pre-flight ships a verdict wherever it can. A request - * nothing can canonicalise is a request nothing can prove identical, and `null` - * matches no other signature, so such a leg stays in the clean-leg set. - */ -const requestSignature = (request: ProbeRequest): string | null => { - try { - return serialize({ ...request, probeId: '' }, PREFLIGHT_ARTIFACT_PATH) - } catch (error) { - if (error instanceof RuntimeFault) return null - throw error - } -} - /** where a leg came from, so a duplicate identifier names its own source. */ type LegOrigin = { readonly leg: PlannedLeg; readonly artifactPath: string } @@ -352,11 +322,6 @@ export const planPreflight: PlanStage = ( const legIdsByOperation = new Map() const scopeKey = (interfaceId: string, operationId: string): string => `${interfaceId}\u0000${operationId}` - // Every planned leg's built request, so the seeded-fault branch below can ask - // whether a leg already in the group carries the fault leg's own. Signatures - // are taken at that site, so a contract seeding no defect canonicalises - // nothing. - const requestByLegId = new Map() const addLeg = ( legId: string, purpose: PlannedLegPurpose, @@ -364,21 +329,21 @@ export const planPreflight: PlanStage = ( operation: AnyOperation, inputs: WitnessInputs, artifactPath: string, - ): PlannedLeg => { - const leg: PlannedLeg = { - legId, - purpose, - request: requestOf(legId, interfaceId, operation, inputs), - operation, - inputs, - } - origins.push({ leg, artifactPath }) - requestByLegId.set(legId, leg.request) + ): void => { + origins.push({ + leg: { + legId, + purpose, + request: requestOf(legId, interfaceId, operation, inputs), + operation, + inputs, + }, + artifactPath, + }) const key = scopeKey(interfaceId, operation.operationId) const group = legIdsByOperation.get(key) if (group === undefined) legIdsByOperation.set(key, [legId]) else group.push(legId) - return leg } // 1. the sensitivity legs and their checks @@ -509,14 +474,14 @@ export const planPreflight: PlanStage = ( `the manifestation witness of ${defect.defectId}`, `${path}.inputs`, ) - // The group is read before the fault leg joins it, which keeps the fault - // leg out of its own clean-leg set. - const group = [ + // Read before the fault leg joins the group, which keeps the fault leg out + // of its own clean-leg set. + const cleanLegIds = [ ...(legIdsByOperation.get( scopeKey(witness.interfaceId, operation.operationId), ) ?? []), ] - const faultLeg = addLeg( + addLeg( witness.legId, 'seeded-fault', witness.interfaceId, @@ -524,40 +489,12 @@ export const planPreflight: PlanStage = ( witness.inputs, `${path}.legId`, ) - // A clean leg has to ask a different question. Another leg of the same - // operation, most often a sensitivity leg, can carry a request equal to - // the fault leg's, and the plan has nothing that tells the two apart: same - // operation, same inputs, same request bytes. A witness firing on such a - // leg says nothing about scope, so it is dropped and the reducer is told - // it was. - // - // Bounded to an operation AD-19 marks as changing no state. That bound is - // caution: nothing here establishes that a mutating operation answers two - // identical requests differently, only that it may, since a request that - // changes state is a different event the second time it is issued. The cost is named in the changelog: a defect seeded on a - // mutating operation whose witness repeats a sensitivity leg's inputs - // still fails this check. - const faultSignature = operation.stateChangeMarker - ? null - : requestSignature(faultLeg.request) - const dropped = - faultSignature === null - ? [] - : group.filter((legId) => { - const request = requestByLegId.get(legId) - return ( - request !== undefined && - requestSignature(request) === faultSignature - ) - }) - const cleanLegIds = group.filter((legId) => !dropped.includes(legId)) checks.push({ kind: 'seeded-faults-scoped', defectId: defect.defectId, witness, operation, cleanLegIds, - droppedLegIds: dropped, }) checks.push({ kind: 'seeded-fault-fired', diff --git a/src/core/preflight/reduce.ts b/src/core/preflight/reduce.ts index 5a68035..73f5201 100644 --- a/src/core/preflight/reduce.ts +++ b/src/core/preflight/reduce.ts @@ -109,6 +109,63 @@ const sameFixtureState = ( ) } +/** + * The canonical digest of one value, or `null` where the value holds something + * RFC 8785 cannot serialise. `JsonValue` admits an integer outside the safe + * range and a lone surrogate, and a 64-bit identifier in a query parameter is + * ordinary, so this is reachable from a contract that parses. A verdict is what + * this stage owes its caller, and `null` compares equal to nothing, so a value + * that cannot be digested leaves the two sides distinguishable and the check + * still reads the leg. + */ +const digestOrNull = (value: unknown): string | null => { + try { + return digestArtifact(value, PREFLIGHT_ARTIFACT_PATH) + } catch (error) { + if (error instanceof RuntimeFault) return null + throw error + } +} + +/** + * Whether two legs issued one request and received one answer, which makes them + * one probe under two labels. A manifestation witness firing on such a leg is + * the fault leg's own manifestation read a second time, and it establishes + * nothing about where the defect is scoped. + * + * Both halves are required. Answers alone would drop AD-10's own worked example, + * two distinct nonexistent identifiers both returning 404: those legs ask + * different questions and are exactly the legs this check exists to read. + * Requests alone are what the plan can see, and identical requests can still be + * answered differently, which is why the comparison lives here where the + * answers are in hand. + * + * The answer half compares the evidence, which is everything a relation can + * address: two legs with equal evidence resolve one relation to one value. It + * carries AD-11's projected body, so a field the operation declares volatile is + * already out of it and a server-minted identifier stops being a difference, + * which is what makes the same request to a mutating operation comparable at + * all. + * + * The correlation identifiers are neutralised on both sides, since they are the + * leg id and differ by construction. A digest that comes back `null` matches + * nothing, so a pair that cannot be compared stays a pair the check reads. + */ +const answeredAlike = (left: LegState, right: LegState): boolean => { + const request = (state: LegState): string | null => + digestOrNull({ ...state.leg.request, probeId: '' }) + const answer = (state: LegState): string | null => + digestOrNull({ ...state.evidence, observationId: '' }) + const leftRequest = request(left) + const leftAnswer = answer(left) + return ( + leftRequest !== null && + leftAnswer !== null && + leftRequest === request(right) && + leftAnswer === answer(right) + ) +} + /** * Resolves a manifestation witness against one leg. Returns `null` when that * leg produced no observation, which the two seeded-fault rows read @@ -306,27 +363,44 @@ export const reducePreflight: ReduceStage< return check(planned.kind, null, 'satisfied', null) } case 'seeded-faults-scoped': { - const { witness, defectId, droppedLegIds } = planned - // A check over no clean leg examined nothing, and a check that - // examined nothing has established nothing, which is the rule the - // `input-sensitivity` row above already runs on. Satisfied here would - // certify scoping from zero evidence, and it would do so on the two - // contracts least able to afford it: one whose defect names the only - // leg its operation has, and one whose every other leg repeats the - // fault leg's request. Those are different authoring mistakes, so the - // note says which one this was. - if (planned.cleanLegIds.length === 0) { - const named = droppedLegIds.map((legId) => `"${legId}"`).join(', ') + const { witness, defectId } = planned + const fault = states.get(witness.legId) + // The legs that answered a different question than the fault leg's, and + // the legs dropped for answering the same one. + const examined: string[] = [] + const dropped: string[] = [] + for (const legId of planned.cleanLegIds) { + const state = states.get(legId) + if ( + state !== undefined && + fault !== undefined && + answeredAlike(state, fault) + ) { + dropped.push(legId) + continue + } + examined.push(legId) + } + // Emptiness is tested on what survived the drop. A check over no clean + // leg examined nothing, and a check that examined nothing has + // established nothing, which is the rule the `input-sensitivity` row + // above already runs on. Satisfied here would certify scoping from zero + // evidence on the three contracts least able to afford it: one whose + // defect names the only leg its operation has, one whose every other leg + // repeats the fault leg's probe, and one where the plan named legs and + // the drop took all of them. The note says which. + if (examined.length === 0) { + const named = dropped.map((legId) => `"${legId}"`).join(', ') return check( planned.kind, witness.operationId, 'failed', - droppedLegIds.length === 0 + dropped.length === 0 ? `${defectId}: the operation has no leg besides the fault leg, so nothing here establishes that the defect is scoped to it` - : `${defectId}: every other leg of the operation carries the fault leg's own request (${named}), so nothing here establishes that the defect is scoped to it`, + : `${defectId}: every other leg of the operation issued the fault leg's own request and received its answer (${named}), so nothing here establishes that the defect is scoped to it`, ) } - for (const legId of planned.cleanLegIds) { + for (const legId of examined) { const resolved = resolveAgainst( witness, states.get(legId), diff --git a/tests/preflight/plan.test.ts b/tests/preflight/plan.test.ts index c648865..3a863bd 100644 --- a/tests/preflight/plan.test.ts +++ b/tests/preflight/plan.test.ts @@ -17,8 +17,6 @@ import { cleanControlProbe, contractDraft, inputsOf, - keylessDefectProbe, - keylessReadContract, parseContract, preflightContract, probeDraft, @@ -332,101 +330,8 @@ describe('the plan as a whole', () => { const scoped = plan.checks.find( (check) => check.kind === 'seeded-faults-scoped', ) - // `list-a` is absent for the separate reason fixture 126 pins: it carries - // the fault leg's own request. What this fixture pins is that no - // `other-api` leg reached the set. expect( scoped?.kind === 'seeded-faults-scoped' ? scoped.cleanLegIds : [], - ).toEqual(['list-b']) - }) - - // The fixture's fault leg reads `list-things` with `limit: 1`, which is the - // sensitivity leg `list-a`'s request exactly. One request gets one answer, so - // a witness firing on `list-a` is the fault's own manifestation read a second - // time. `list-b` asks for `limit: 2` and stays. - it("126. drops a clean leg carrying the fault leg's own request, and keeps one that differs", () => { - const scoped = planOf().checks.find( - (check) => check.kind === 'seeded-faults-scoped', - ) - if (scoped?.kind !== 'seeded-faults-scoped') - throw new Error('the fixture declares one seeded defect') - expect(scoped.cleanLegIds).toEqual(['list-b']) - const requestOfLeg = (legId: string) => - planOf().legs.find((leg) => leg.legId === legId)?.request.channels - expect(requestOfLeg('list-a')).toEqual(requestOfLeg('fault-leg')) - expect(requestOfLeg('list-b')).not.toEqual(requestOfLeg('fault-leg')) - }) - - // The exclusion holds where one request has one answer. `create-thing` - // declares `stateChangeMarker: true`, so the same body posted twice is two - // events the environment may answer differently, and both legs stay. - it("128. keeps a clean leg carrying the fault leg's request when the operation changes state", () => { - const draft = probeDraft() - draft.defects[0].manifestationWitness.operationId = 'create-thing' - draft.defects[0].manifestationWitness.inputs = inputsOf({ - body: { kind: 'json', value: { name: 'alpha' } }, - }) - const plan = planOf(preflightContract, [ProbeSchema.parse(draft)]) - const scoped = plan.checks.find( - (check) => check.kind === 'seeded-faults-scoped', - ) - if (scoped?.kind !== 'seeded-faults-scoped') - throw new Error('the draft declares one seeded defect') - expect(scoped.cleanLegIds).toEqual(['create-a', 'create-b']) - }) - - // AD-10's exemption case: one keyless safe read, whose only legs are the two - // control-observe legs, both carrying the empty inputs the operation admits. - // A defect seeded there matches every leg it has. The reducer is told which - // legs went, because an empty set it read as satisfied would certify scoping - // from nothing (fixture 132). - it("130. names the legs it dropped when the fault leg's request matches every leg of its operation", () => { - const plan = planPreflight({ - contract: keylessReadContract, - probes: [keylessDefectProbe], - runId: 'run-1', - }) - const scoped = plan.checks.find( - (check) => check.kind === 'seeded-faults-scoped', - ) - if (scoped?.kind !== 'seeded-faults-scoped') - throw new Error('the probe declares one seeded defect') - expect(scoped.cleanLegIds).toEqual([]) - expect(scoped.droppedLegIds).toEqual([ - 'preflight-control-observe', - 'preflight-control-observe-2', - ]) - }) - - // `JsonValue` admits an integer outside the safe range and RFC 8785 does not, - // so the signature of such a request is unavailable. Planning stays a plan: - // no fault is thrown, and a leg that cannot be proved identical stays in the - // clean-leg set. - it('131. keeps a leg whose request holds a value RFC 8785 cannot serialise', () => { - const draft = contractDraft() - const listThings = draft.permittedInterfaces[0].operations.find( - (operation: { operationId: string }) => - operation.operationId === 'list-things', - ) - listThings.sensitivityWitness.legs[0].inputs.query = { limit: 1e21 } - const cleanLegsOf = (probes: readonly Probe[]) => { - const scoped = planOf(parseContract(draft), probes).checks.find( - (check) => check.kind === 'seeded-faults-scoped', - ) - if (scoped?.kind !== 'seeded-faults-scoped') - throw new Error('the probe declares one seeded defect') - return scoped.cleanLegIds - } - // The unserialisable leg is the clean one here. - expect(cleanLegsOf([seededProbe])).toEqual(['list-a', 'list-b']) - // And here it is the fault leg, so no leg is dropped at all. - const faulty = probeDraft() - faulty.defects[0].manifestationWitness.inputs = inputsOf({ - query: { limit: 1e21 }, - }) - expect(cleanLegsOf([ProbeSchema.parse(faulty)])).toEqual([ - 'list-a', - 'list-b', - ]) + ).toEqual(['list-a', 'list-b']) }) }) diff --git a/tests/preflight/reduce.test.ts b/tests/preflight/reduce.test.ts index 48bf364..485cde0 100644 --- a/tests/preflight/reduce.test.ts +++ b/tests/preflight/reduce.test.ts @@ -314,34 +314,120 @@ describe('the two seeded-fault checks, which are disjoint by construction', () = ).toBe('failed') }) - // `list-a` carries the fault leg's request byte for byte, so an environment - // that answered one that way answered the other the same way. The plan drops - // such a leg (fixture 126) and the check stays satisfied; without that, every - // contract whose sensitivity legs cover the witness's own inputs failed - // pre-flight on one observation counted twice. - it("127. seeded-faults-scoped stays satisfied when the leg the witness fires on carries the fault leg's own request", () => { + // `list-a` reads `list-things` with `limit: 1`, which is the fault leg's + // request byte for byte. Answered the same way too, the two legs are one + // probe under two labels, and a witness firing there is the fault leg's own + // manifestation read a second time. + it("126. seeded-faults-scoped drops a clean leg that issued the fault leg's request and got its answer", () => { + expect( + outcomeOf( + { + patches: { + 'list-a': jsonPatch({ items: [{ id: 'r-1', broken: true }] }), + }, + }, + 'seeded-faults-scoped', + 'list-things', + ), + ).toBe('satisfied') + }) + + // The same request, answered differently. That is a second answer and it is + // evidence of its own, so the leg is read and the check fails on it. + it("127. seeded-faults-scoped fails on a leg that issued the fault leg's request and got a different answer", () => { expect( outcomeOf( { patches: { 'list-a': jsonPatch({ items: [{ broken: true }] }) } }, 'seeded-faults-scoped', 'list-things', ), + ).toBe('failed') + }) + + // The mutating case, which is the one the plan could never decide. `create-a` + // posts the body the witness posts, and the answers agree once `/id`, which + // the operation declares volatile, is out of the projection. + it('128. seeded-faults-scoped drops a clean leg of a mutating operation answered as the fault leg was', () => { + expect( + outcomeOf( + { + probes: [mutatingDefectProbe()], + patches: { + 'fault-leg': jsonPatch({ id: 'x-9', ok: true, echo: 'alpha' }), + }, + }, + 'seeded-faults-scoped', + 'create-thing', + ), ).toBe('satisfied') }) - // An empty clean-leg set examined nothing, so it establishes nothing. The two - // ways it empties are different authoring mistakes and the note says which. - it("132. seeded-faults-scoped fails when every other leg of the operation carries the fault leg's request", () => { + it('132. seeded-faults-scoped fails on a clean leg of a mutating operation answered differently', () => { + expect( + outcomeOf( + { + probes: [mutatingDefectProbe()], + patches: { + 'fault-leg': jsonPatch({ id: 'x-9', ok: true, echo: 'beta' }), + }, + }, + 'seeded-faults-scoped', + 'create-thing', + ), + ).toBe('failed') + }) + + // AD-10's exemption case: one keyless safe read, whose only legs are the two + // control-observe legs, both sending the empty inputs the operation admits + // and both answered alike. Every leg the plan named is dropped, and emptiness + // is tested on what survived, so the check fails and names the cause. + it('130. seeded-faults-scoped fails when the drop takes every leg the plan named', () => { + const answer = jsonPatch({ ok: false }) const { checks } = verdictOf({ contract: keylessReadContract, probes: [keylessDefectProbe], + patches: { + 'preflight-control-observe': answer, + 'preflight-control-observe-2': answer, + 'fault-leg': answer, + }, }) const scoped = checkFor(checks, 'seeded-faults-scoped', 'read-health') expect(scoped.outcome).toBe('failed') - expect(scoped.note).toContain("carries the fault leg's own request") + expect(scoped.note).toContain( + "issued the fault leg's own request and received its answer", + ) expect(scoped.note).toContain('"preflight-control-observe"') }) + // `JsonValue` admits an integer outside the safe range and RFC 8785 does not. + // Both legs here send it, so both digests come back unavailable. A pair that + // cannot be compared is a pair the check reads, and the reducer answers with a + // verdict; the fault stays inside the comparison. + it('131. seeded-faults-scoped reads a leg whose request holds a value RFC 8785 cannot serialise', () => { + const draft = contractDraft() + const listThings = draft.permittedInterfaces[0].operations.find( + (operation: { operationId: string }) => + operation.operationId === 'list-things', + ) + listThings.sensitivityWitness.legs[0].inputs.query = { limit: 1e21 } + const probe = probeDraft() + probe.defects[0].manifestationWitness.inputs.query = { limit: 1e21 } + expect( + outcomeOf( + { + contract: parseContract(draft), + probes: [ProbeSchema.parse(probe)], + patches: { + 'list-a': jsonPatch({ items: [{ id: 'r-1', broken: true }] }), + }, + }, + 'seeded-faults-scoped', + 'list-things', + ), + ).toBe('failed') + }) + it('133. seeded-faults-scoped fails when the operation has no leg besides the fault leg', () => { const scoped = checkFor( verdictOf({ probes: [lonelyDefectProbe] }).checks, @@ -549,6 +635,30 @@ function driftPatches(): Record { } } +/** + * The seeded probe aimed at the mutating operation, posting the body sensitivity + * leg `create-a` posts, with a relation that reads the echo `create-thing` + * returns. + */ +function mutatingDefectProbe(): Probe { + const draft = probeDraft() + draft.defects[0].manifestationWitness.operationId = 'create-thing' + draft.defects[0].manifestationWitness.inputs = { + path: {}, + query: {}, + header: {}, + body: { kind: 'json', value: { name: 'alpha' } }, + } + draft.defects[0].manifestationWitness.relation = { + op: 'equality', + operands: [ + { pointer: '/interactions/fault-leg/response-body/echo' }, + { literal: 'alpha' }, + ], + } + return ProbeSchema.parse(draft) +} + /** the seeded fault failing to fire on its own leg. */ function faultSilentPatch(): Record { return { 'fault-leg': jsonPatch({ items: [{ id: 'r-1', broken: false }] }) }