From 001efe7a71b26b4e199aed32b67d397760acc08b Mon Sep 17 00:00:00 2001 From: flyworker Date: Sat, 5 Sep 2026 16:31:20 +0000 Subject: [PATCH] feat(inference): declare how a context window was determined MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The agent already distinguishes a window read from the backend's max_model_len from one asserted in models.json, but sent only the number. A receiver could not tell them apart, and they are not equally trustworthy. A detection is self-verifying: vLLM and SGLang refuse to start when --max-model-len exceeds what the KV cache can hold, so a declared window is backed by memory that demonstrably exists. An override is an assertion nothing on the agent side can check. That distinction starts to matter once share of traffic is weighted by the declared window, because the override path is the one with an incentive attached. context_source now travels with the value on both the heartbeat metadata and the schema-1 declaration, carrying "detected" or "override". An absent field means unknown, deliberately. Agents that do not send it read exactly as they did before, so this needs no coordinated rollout, and "unknown" and "pending" are not sent explicitly — that is what absent already conveys, and emitting them would add a value every receiver has to special-case. A source is only ever attached to a window that was actually determined. The provider now returns the window and its provenance together rather than just an int, so the two cannot disagree. --- docs/configuration.md | 8 ++ internal/computing/context_provenance_test.go | 111 ++++++++++++++++++ internal/computing/inference_client.go | 60 ++++++++-- internal/computing/inference_service.go | 19 +-- .../computing/model_context_source_test.go | 5 +- internal/computing/model_context_test.go | 14 ++- 6 files changed, 190 insertions(+), 27 deletions(-) create mode 100644 internal/computing/context_provenance_test.go diff --git a/docs/configuration.md b/docs/configuration.md index 4ba21859..92fd3685 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -108,6 +108,14 @@ Back up `private_key` somewhere safe. It is 32 bytes and irreplaceable. `computing-provider inference status` prints the window and source per model, and the provider logs a warning once per model when nothing is reported. +The source travels with the value to Swan Inference as `context_source`, because +the two paths are not equally trustworthy and the number alone does not say +which produced it. A **detected** window is self-verifying — vLLM and SGLang +refuse to start when it exceeds what the KV cache can hold, so it is backed by +memory that demonstrably exists. An **override** is an assertion nothing on this +side can check. An absent `context_source` means unknown, so an agent that does +not send one reads exactly as it did before. + ### Self-check and auto-heal The provider audits itself on a timer and, when a backend cannot serve, takes the model out of routing until it can again. diff --git a/internal/computing/context_provenance_test.go b/internal/computing/context_provenance_test.go new file mode 100644 index 00000000..1513ace7 --- /dev/null +++ b/internal/computing/context_provenance_test.go @@ -0,0 +1,111 @@ +package computing + +import ( + "encoding/json" + "strings" + "testing" +) + +// An absent field must mean "unknown". Receivers that predate the field, and +// agents that cannot determine a window, have to keep reading identically — +// which is what lets this ship without a coordinated rollout. +func TestContextSourceOmittedWhenNotDetermined(t *testing.T) { + for _, tc := range []struct { + name string + info ModelContextInfo + want string + }{ + {"detected", ModelContextInfo{Length: 32768, Source: ContextSourceDetected}, ContextSourceDetected}, + {"override", ModelContextInfo{Length: 128000, Source: ContextSourceOverride}, ContextSourceOverride}, + {"unknown is absent", ModelContextInfo{Length: 0, Source: ContextSourceUnknown}, ""}, + {"pending is absent", ModelContextInfo{Length: 0, Source: ContextSourcePending}, ""}, + // A source without a length has nothing whose provenance could matter. + {"source without length is absent", ModelContextInfo{Length: 0, Source: ContextSourceDetected}, ""}, + } { + if got := declaredContextSource(tc.info); got != tc.want { + t.Errorf("%s: got %q, want %q", tc.name, got, tc.want) + } + } +} + +// The JSON must actually omit the key, not send an empty string — "" is a +// present field and a receiver would have to special-case it. +func TestContextSourceAbsentFromJSONWhenUnknown(t *testing.T) { + unknown, err := json.Marshal(ModelInfo{ModelID: "m", ContextLength: 0}) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(unknown), "context_source") { + t.Errorf("unknown window emitted the key: %s", unknown) + } + + known, err := json.Marshal(ModelInfo{ModelID: "m", ContextLength: 32768, ContextSource: ContextSourceDetected}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(known), `"context_source":"detected"`) { + t.Errorf("detected window did not carry its provenance: %s", known) + } +} + +func TestHeartbeatMetadataCarriesProvenance(t *testing.T) { + c := &InferenceClient{models: []string{"detected-model", "override-model", "unknown-model"}} + c.SetModelContextsProvider(func() map[string]ModelContextInfo { + return map[string]ModelContextInfo{ + "detected-model": {Length: 32768, Source: ContextSourceDetected}, + "override-model": {Length: 128000, Source: ContextSourceOverride}, + // unknown-model deliberately absent + } + }) + + byID := map[string]ModelInfo{} + for _, info := range c.buildModelMetadata() { + byID[info.ModelID] = info + } + + if got := byID["detected-model"]; got.ContextLength != 32768 || got.ContextSource != ContextSourceDetected { + t.Errorf("detected model = %+v", got) + } + if got := byID["override-model"]; got.ContextLength != 128000 || got.ContextSource != ContextSourceOverride { + t.Errorf("override model = %+v", got) + } + if _, ok := byID["unknown-model"]; ok { + t.Error("a model with no determined window should not be declared at all") + } +} + +func TestDeclarationCarriesProvenance(t *testing.T) { + c := &InferenceClient{models: []string{"detected-model", "unknown-model"}} + c.SetModelContextsProvider(func() map[string]ModelContextInfo { + return map[string]ModelContextInfo{ + "detected-model": {Length: 32768, Source: ContextSourceDetected}, + } + }) + c.SetModelMappingsProvider(func() map[string]ModelMapping { + return map[string]ModelMapping{ + "detected-model": {Category: "text-generation"}, + "unknown-model": {Category: "text-generation"}, + } + }) + + byID := map[string]ModelDeclaration{} + for _, d := range c.buildModelDeclarations() { + byID[d.ModelID] = d + } + + if got := byID["detected-model"].ContextSource; got != ContextSourceDetected { + t.Errorf("declaration source = %q, want %q", got, ContextSourceDetected) + } + if got := byID["unknown-model"].ContextSource; got != "" { + t.Errorf("undetermined window declared a source %q, want none", got) + } + + // And the same must hold once serialised. + raw, err := json.Marshal(byID["unknown-model"]) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(raw), "context_source") { + t.Errorf("undetermined declaration emitted the key: %s", raw) + } +} diff --git a/internal/computing/inference_client.go b/internal/computing/inference_client.go index 30071d98..b4abcfaa 100644 --- a/internal/computing/inference_client.go +++ b/internal/computing/inference_client.go @@ -71,6 +71,17 @@ type ModelInfo struct { Format string `json:"format,omitempty"` // Weight format: "fp16", "fp8", "awq", "gptq", "gguf" Quantization string `json:"quantization,omitempty"` // Quantization detail: "q4_k_m", "q8_0", "w4a16", etc. ContextLength int `json:"context_length,omitempty"` // Backend's real context window in tokens (#61); 0 = unknown, server assumes catalog value + // ContextSource says how ContextLength was arrived at: "detected" read + // from the backend's own max_model_len, or "override" asserted in + // models.json. Absent means unknown, so an agent that does not send it + // reads exactly as it did before — no coordinated rollout required. + // + // The two are not equally trustworthy and a receiver cannot tell them + // apart from the number alone. A detection is self-verifying: vLLM and + // SGLang refuse to start when the window exceeds what the KV cache can + // hold, so the value is backed by memory that demonstrably exists. An + // override is an assertion nothing on this side can check. + ContextSource string `json:"context_source,omitempty"` } // VerifyResponsePayload is returned after processing a verification challenge @@ -297,9 +308,9 @@ type InferenceClient struct { streamingInferenceHandler StreamingInferenceHandler warmupHandler WarmupHandler noticeHandler NoticeHandler - modelHealthProvider func() map[string]string // Returns current model health for heartbeat - modelMappingsProvider func() map[string]ModelMapping // Returns current model mappings for format/quantization - modelContextsProvider func() map[string]int // Returns per-model real context windows for register/heartbeat (#61) + modelHealthProvider func() map[string]string // Returns current model health for heartbeat + modelMappingsProvider func() map[string]ModelMapping // Returns current model mappings for format/quantization + modelContextsProvider func() map[string]ModelContextInfo // Per-model real context window and how it was determined (#61) mu sync.RWMutex writeMu sync.Mutex // Mutex for WebSocket writes to prevent concurrent writes @@ -416,7 +427,7 @@ func (c *InferenceClient) SetModelMappingsProvider(provider func() map[string]Mo // SetModelContextsProvider sets the function that returns each model's real // context window (manual override or backend-detected) for register/heartbeat -func (c *InferenceClient) SetModelContextsProvider(provider func() map[string]int) { +func (c *InferenceClient) SetModelContextsProvider(provider func() map[string]ModelContextInfo) { c.modelContextsProvider = provider } @@ -2093,11 +2104,30 @@ func (c *InferenceClient) sendBenchmarkResponse(requestID, benchmarkID string, r } } +// declaredContextSource is the provenance to put on the wire for a context +// window, or "" to leave the field off. +// +// Only a window that was actually determined carries one: with no length there +// is nothing whose provenance could matter, and "unknown" and "pending" are +// what an absent field already means. Sending them explicitly would add a value +// every receiver has to special-case for no gain. +func declaredContextSource(info ModelContextInfo) string { + if info.Length <= 0 { + return "" + } + switch info.Source { + case ContextSourceOverride, ContextSourceDetected: + return info.Source + default: + return "" + } +} + // buildModelMetadata builds the lightweight per-model metadata list for // heartbeats: context window (manual or backend-detected), format, and // quantization — without touching hash manifests on disk (#61). func (c *InferenceClient) buildModelMetadata() []ModelInfo { - var contexts map[string]int + var contexts map[string]ModelContextInfo if c.modelContextsProvider != nil { contexts = c.modelContextsProvider() } @@ -2110,7 +2140,8 @@ func (c *InferenceClient) buildModelMetadata() []ModelInfo { for _, modelID := range c.models { info := ModelInfo{ ModelID: modelID, - ContextLength: contexts[modelID], + ContextLength: contexts[modelID].Length, + ContextSource: declaredContextSource(contexts[modelID]), } if mapping, ok := mappings[modelID]; ok { info.Format = mapping.Format @@ -2134,7 +2165,7 @@ func (c *InferenceClient) loadModelHashes() []ModelInfo { } // Get real per-model context windows (manual override or backend-detected) - var contexts map[string]int + var contexts map[string]ModelContextInfo if c.modelContextsProvider != nil { contexts = c.modelContextsProvider() } @@ -2149,7 +2180,8 @@ func (c *InferenceClient) loadModelHashes() []ModelInfo { info.Format = mapping.Format info.Quantization = mapping.Quantization } - info.ContextLength = contexts[modelID] + info.ContextLength = contexts[modelID].Length + info.ContextSource = declaredContextSource(contexts[modelID]) modelDir := c.getModelDir(modelID) manifest, err := models.LoadHashManifest(modelDir) @@ -2392,8 +2424,11 @@ type DeclaredCapacity struct { } type ModelDeclaration struct { - SchemaVersion string `json:"schema_version"` - ModelID string `json:"model_id"` + SchemaVersion string `json:"schema_version"` + ModelID string `json:"model_id"` + // ContextSource is the provenance of max_context_length below: "detected" + // or "override". Absent means unknown. + ContextSource string `json:"context_source,omitempty"` WeightHash string `json:"weight_hash,omitempty"` HashAlgo string `json:"hash_algo,omitempty"` Quantization string `json:"quantization,omitempty"` @@ -2487,7 +2522,7 @@ func declaredModalitiesFor(category string, contextLen int) ([]DeclaredModality, // declared. Over-claiming is what the verification programme is designed to // catch, and there is nothing to gain by it. func (c *InferenceClient) buildModelDeclarations() []ModelDeclaration { - var contexts map[string]int + var contexts map[string]ModelContextInfo if c.modelContextsProvider != nil { contexts = c.modelContextsProvider() } @@ -2499,11 +2534,12 @@ func (c *InferenceClient) buildModelDeclarations() []ModelDeclaration { decls := make([]ModelDeclaration, 0, len(c.models)) for _, modelID := range c.models { mapping := mappings[modelID] - in, out := declaredModalitiesFor(mapping.Category, contexts[modelID]) + in, out := declaredModalitiesFor(mapping.Category, contexts[modelID].Length) d := ModelDeclaration{ SchemaVersion: ModelDeclarationSchemaVersion, ModelID: modelID, + ContextSource: declaredContextSource(contexts[modelID]), Quantization: normalizeQuantization(mapping.Format, mapping.Quantization), Engine: detectEngineName(mapping), InputModalities: in, diff --git a/internal/computing/inference_service.go b/internal/computing/inference_service.go index 5b54cc19..de94c36f 100644 --- a/internal/computing/inference_service.go +++ b/internal/computing/inference_service.go @@ -826,16 +826,19 @@ func (s *InferenceService) handleStreamingInference(requestID string, payload In return s.streamFromDockerModel(endpoint, payload.Request, payload.ModelID, localModel, apiKey, sendChunk) } -// resolveModelContexts returns each configured model's real context window in -// tokens: a manual context_length from models.json wins; otherwise the value -// the health checker detected from the backend's /v1/models (max_model_len). -// Models with no known window are omitted — the server falls back to the -// catalog value for those (#61). -func (s *InferenceService) resolveModelContexts() map[string]int { - contexts := make(map[string]int) +// resolveModelContexts returns each configured model's real context window and +// how it was determined: a manual context_length from models.json wins; +// otherwise the value the health checker detected from the backend's +// /v1/models (max_model_len). Models with no known window are omitted — the +// server falls back to the catalog value for those (#61). +// +// The provenance travels with the value because the two paths are not equally +// trustworthy, and the number alone does not say which one produced it. +func (s *InferenceService) resolveModelContexts() map[string]ModelContextInfo { + contexts := make(map[string]ModelContextInfo) for modelID := range s.modelMappings { if info := s.ModelContext(modelID); info.Length > 0 { - contexts[modelID] = info.Length + contexts[modelID] = info } else if info.Source == ContextSourceUnknown { s.warnUnknownContext(modelID) } diff --git a/internal/computing/model_context_source_test.go b/internal/computing/model_context_source_test.go index 3d103db0..0fa16650 100644 --- a/internal/computing/model_context_source_test.go +++ b/internal/computing/model_context_source_test.go @@ -65,9 +65,12 @@ func TestResolveOmitsUnknownContext(t *testing.T) { s.healthChecker.recordDetectedContext("org/unknown", nil) got := s.resolveModelContexts() - if len(got) != 1 || got["org/known"] != 8192 { + if len(got) != 1 || got["org/known"].Length != 8192 { t.Fatalf("got %v, want only org/known=8192", got) } + if got["org/known"].Source != ContextSourceDetected { + t.Errorf("source = %q, want %q", got["org/known"].Source, ContextSourceDetected) + } } // The warning is the whole point of the change, but it must not repeat: this diff --git a/internal/computing/model_context_test.go b/internal/computing/model_context_test.go index 1f84b635..c95b2019 100644 --- a/internal/computing/model_context_test.go +++ b/internal/computing/model_context_test.go @@ -83,8 +83,10 @@ func TestRecordDetectedContextLocalNameMatch(t *testing.T) { func TestBuildModelMetadataHeartbeatShape(t *testing.T) { c := &InferenceClient{models: []string{"model-a", "model-b", "model-c"}} - c.SetModelContextsProvider(func() map[string]int { - return map[string]int{"model-a": 32768} + c.SetModelContextsProvider(func() map[string]ModelContextInfo { + return map[string]ModelContextInfo{ + "model-a": {Length: 32768, Source: ContextSourceDetected}, + } }) c.SetModelMappingsProvider(func() map[string]ModelMapping { return map[string]ModelMapping{ @@ -126,11 +128,11 @@ func TestResolveModelContextsPrecedence(t *testing.T) { } contexts := s.resolveModelContexts() - if contexts["model-a"] != 32768 { - t.Errorf("expected detected 32768 for model-a, got %d", contexts["model-a"]) + if contexts["model-a"].Length != 32768 || contexts["model-a"].Source != ContextSourceDetected { + t.Errorf("expected detected 32768 for model-a, got %+v", contexts["model-a"]) } - if contexts["model-b"] != 16384 { - t.Errorf("expected manual override 16384 for model-b, got %d", contexts["model-b"]) + if contexts["model-b"].Length != 16384 || contexts["model-b"].Source != ContextSourceOverride { + t.Errorf("expected override 16384 for model-b, got %+v", contexts["model-b"]) } if _, ok := contexts["model-c"]; ok { t.Error("model with unknown context should be omitted")