diff --git a/docs/configuration.md b/docs/configuration.md index 4ba2185..92fd368 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -108,6 +108,14 @@ Back up `private_key` somewhere safe. It is 32 bytes and irreplaceable. `computing-provider inference status` prints the window and source per model, and the provider logs a warning once per model when nothing is reported. +The source travels with the value to Swan Inference as `context_source`, because +the two paths are not equally trustworthy and the number alone does not say +which produced it. A **detected** window is self-verifying — vLLM and SGLang +refuse to start when it exceeds what the KV cache can hold, so it is backed by +memory that demonstrably exists. An **override** is an assertion nothing on this +side can check. An absent `context_source` means unknown, so an agent that does +not send one reads exactly as it did before. + ### Self-check and auto-heal The provider audits itself on a timer and, when a backend cannot serve, takes the model out of routing until it can again. diff --git a/internal/computing/context_provenance_test.go b/internal/computing/context_provenance_test.go new file mode 100644 index 0000000..1513ace --- /dev/null +++ b/internal/computing/context_provenance_test.go @@ -0,0 +1,111 @@ +package computing + +import ( + "encoding/json" + "strings" + "testing" +) + +// An absent field must mean "unknown". Receivers that predate the field, and +// agents that cannot determine a window, have to keep reading identically — +// which is what lets this ship without a coordinated rollout. +func TestContextSourceOmittedWhenNotDetermined(t *testing.T) { + for _, tc := range []struct { + name string + info ModelContextInfo + want string + }{ + {"detected", ModelContextInfo{Length: 32768, Source: ContextSourceDetected}, ContextSourceDetected}, + {"override", ModelContextInfo{Length: 128000, Source: ContextSourceOverride}, ContextSourceOverride}, + {"unknown is absent", ModelContextInfo{Length: 0, Source: ContextSourceUnknown}, ""}, + {"pending is absent", ModelContextInfo{Length: 0, Source: ContextSourcePending}, ""}, + // A source without a length has nothing whose provenance could matter. + {"source without length is absent", ModelContextInfo{Length: 0, Source: ContextSourceDetected}, ""}, + } { + if got := declaredContextSource(tc.info); got != tc.want { + t.Errorf("%s: got %q, want %q", tc.name, got, tc.want) + } + } +} + +// The JSON must actually omit the key, not send an empty string — "" is a +// present field and a receiver would have to special-case it. +func TestContextSourceAbsentFromJSONWhenUnknown(t *testing.T) { + unknown, err := json.Marshal(ModelInfo{ModelID: "m", ContextLength: 0}) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(unknown), "context_source") { + t.Errorf("unknown window emitted the key: %s", unknown) + } + + known, err := json.Marshal(ModelInfo{ModelID: "m", ContextLength: 32768, ContextSource: ContextSourceDetected}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(known), `"context_source":"detected"`) { + t.Errorf("detected window did not carry its provenance: %s", known) + } +} + +func TestHeartbeatMetadataCarriesProvenance(t *testing.T) { + c := &InferenceClient{models: []string{"detected-model", "override-model", "unknown-model"}} + c.SetModelContextsProvider(func() map[string]ModelContextInfo { + return map[string]ModelContextInfo{ + "detected-model": {Length: 32768, Source: ContextSourceDetected}, + "override-model": {Length: 128000, Source: ContextSourceOverride}, + // unknown-model deliberately absent + } + }) + + byID := map[string]ModelInfo{} + for _, info := range c.buildModelMetadata() { + byID[info.ModelID] = info + } + + if got := byID["detected-model"]; got.ContextLength != 32768 || got.ContextSource != ContextSourceDetected { + t.Errorf("detected model = %+v", got) + } + if got := byID["override-model"]; got.ContextLength != 128000 || got.ContextSource != ContextSourceOverride { + t.Errorf("override model = %+v", got) + } + if _, ok := byID["unknown-model"]; ok { + t.Error("a model with no determined window should not be declared at all") + } +} + +func TestDeclarationCarriesProvenance(t *testing.T) { + c := &InferenceClient{models: []string{"detected-model", "unknown-model"}} + c.SetModelContextsProvider(func() map[string]ModelContextInfo { + return map[string]ModelContextInfo{ + "detected-model": {Length: 32768, Source: ContextSourceDetected}, + } + }) + c.SetModelMappingsProvider(func() map[string]ModelMapping { + return map[string]ModelMapping{ + "detected-model": {Category: "text-generation"}, + "unknown-model": {Category: "text-generation"}, + } + }) + + byID := map[string]ModelDeclaration{} + for _, d := range c.buildModelDeclarations() { + byID[d.ModelID] = d + } + + if got := byID["detected-model"].ContextSource; got != ContextSourceDetected { + t.Errorf("declaration source = %q, want %q", got, ContextSourceDetected) + } + if got := byID["unknown-model"].ContextSource; got != "" { + t.Errorf("undetermined window declared a source %q, want none", got) + } + + // And the same must hold once serialised. + raw, err := json.Marshal(byID["unknown-model"]) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(raw), "context_source") { + t.Errorf("undetermined declaration emitted the key: %s", raw) + } +} diff --git a/internal/computing/inference_client.go b/internal/computing/inference_client.go index 30071d9..b4abcfa 100644 --- a/internal/computing/inference_client.go +++ b/internal/computing/inference_client.go @@ -71,6 +71,17 @@ type ModelInfo struct { Format string `json:"format,omitempty"` // Weight format: "fp16", "fp8", "awq", "gptq", "gguf" Quantization string `json:"quantization,omitempty"` // Quantization detail: "q4_k_m", "q8_0", "w4a16", etc. ContextLength int `json:"context_length,omitempty"` // Backend's real context window in tokens (#61); 0 = unknown, server assumes catalog value + // ContextSource says how ContextLength was arrived at: "detected" read + // from the backend's own max_model_len, or "override" asserted in + // models.json. Absent means unknown, so an agent that does not send it + // reads exactly as it did before — no coordinated rollout required. + // + // The two are not equally trustworthy and a receiver cannot tell them + // apart from the number alone. A detection is self-verifying: vLLM and + // SGLang refuse to start when the window exceeds what the KV cache can + // hold, so the value is backed by memory that demonstrably exists. An + // override is an assertion nothing on this side can check. + ContextSource string `json:"context_source,omitempty"` } // VerifyResponsePayload is returned after processing a verification challenge @@ -297,9 +308,9 @@ type InferenceClient struct { streamingInferenceHandler StreamingInferenceHandler warmupHandler WarmupHandler noticeHandler NoticeHandler - modelHealthProvider func() map[string]string // Returns current model health for heartbeat - modelMappingsProvider func() map[string]ModelMapping // Returns current model mappings for format/quantization - modelContextsProvider func() map[string]int // Returns per-model real context windows for register/heartbeat (#61) + modelHealthProvider func() map[string]string // Returns current model health for heartbeat + modelMappingsProvider func() map[string]ModelMapping // Returns current model mappings for format/quantization + modelContextsProvider func() map[string]ModelContextInfo // Per-model real context window and how it was determined (#61) mu sync.RWMutex writeMu sync.Mutex // Mutex for WebSocket writes to prevent concurrent writes @@ -416,7 +427,7 @@ func (c *InferenceClient) SetModelMappingsProvider(provider func() map[string]Mo // SetModelContextsProvider sets the function that returns each model's real // context window (manual override or backend-detected) for register/heartbeat -func (c *InferenceClient) SetModelContextsProvider(provider func() map[string]int) { +func (c *InferenceClient) SetModelContextsProvider(provider func() map[string]ModelContextInfo) { c.modelContextsProvider = provider } @@ -2093,11 +2104,30 @@ func (c *InferenceClient) sendBenchmarkResponse(requestID, benchmarkID string, r } } +// declaredContextSource is the provenance to put on the wire for a context +// window, or "" to leave the field off. +// +// Only a window that was actually determined carries one: with no length there +// is nothing whose provenance could matter, and "unknown" and "pending" are +// what an absent field already means. Sending them explicitly would add a value +// every receiver has to special-case for no gain. +func declaredContextSource(info ModelContextInfo) string { + if info.Length <= 0 { + return "" + } + switch info.Source { + case ContextSourceOverride, ContextSourceDetected: + return info.Source + default: + return "" + } +} + // buildModelMetadata builds the lightweight per-model metadata list for // heartbeats: context window (manual or backend-detected), format, and // quantization — without touching hash manifests on disk (#61). func (c *InferenceClient) buildModelMetadata() []ModelInfo { - var contexts map[string]int + var contexts map[string]ModelContextInfo if c.modelContextsProvider != nil { contexts = c.modelContextsProvider() } @@ -2110,7 +2140,8 @@ func (c *InferenceClient) buildModelMetadata() []ModelInfo { for _, modelID := range c.models { info := ModelInfo{ ModelID: modelID, - ContextLength: contexts[modelID], + ContextLength: contexts[modelID].Length, + ContextSource: declaredContextSource(contexts[modelID]), } if mapping, ok := mappings[modelID]; ok { info.Format = mapping.Format @@ -2134,7 +2165,7 @@ func (c *InferenceClient) loadModelHashes() []ModelInfo { } // Get real per-model context windows (manual override or backend-detected) - var contexts map[string]int + var contexts map[string]ModelContextInfo if c.modelContextsProvider != nil { contexts = c.modelContextsProvider() } @@ -2149,7 +2180,8 @@ func (c *InferenceClient) loadModelHashes() []ModelInfo { info.Format = mapping.Format info.Quantization = mapping.Quantization } - info.ContextLength = contexts[modelID] + info.ContextLength = contexts[modelID].Length + info.ContextSource = declaredContextSource(contexts[modelID]) modelDir := c.getModelDir(modelID) manifest, err := models.LoadHashManifest(modelDir) @@ -2392,8 +2424,11 @@ type DeclaredCapacity struct { } type ModelDeclaration struct { - SchemaVersion string `json:"schema_version"` - ModelID string `json:"model_id"` + SchemaVersion string `json:"schema_version"` + ModelID string `json:"model_id"` + // ContextSource is the provenance of max_context_length below: "detected" + // or "override". Absent means unknown. + ContextSource string `json:"context_source,omitempty"` WeightHash string `json:"weight_hash,omitempty"` HashAlgo string `json:"hash_algo,omitempty"` Quantization string `json:"quantization,omitempty"` @@ -2487,7 +2522,7 @@ func declaredModalitiesFor(category string, contextLen int) ([]DeclaredModality, // declared. Over-claiming is what the verification programme is designed to // catch, and there is nothing to gain by it. func (c *InferenceClient) buildModelDeclarations() []ModelDeclaration { - var contexts map[string]int + var contexts map[string]ModelContextInfo if c.modelContextsProvider != nil { contexts = c.modelContextsProvider() } @@ -2499,11 +2534,12 @@ func (c *InferenceClient) buildModelDeclarations() []ModelDeclaration { decls := make([]ModelDeclaration, 0, len(c.models)) for _, modelID := range c.models { mapping := mappings[modelID] - in, out := declaredModalitiesFor(mapping.Category, contexts[modelID]) + in, out := declaredModalitiesFor(mapping.Category, contexts[modelID].Length) d := ModelDeclaration{ SchemaVersion: ModelDeclarationSchemaVersion, ModelID: modelID, + ContextSource: declaredContextSource(contexts[modelID]), Quantization: normalizeQuantization(mapping.Format, mapping.Quantization), Engine: detectEngineName(mapping), InputModalities: in, diff --git a/internal/computing/inference_service.go b/internal/computing/inference_service.go index 5b54cc1..de94c36 100644 --- a/internal/computing/inference_service.go +++ b/internal/computing/inference_service.go @@ -826,16 +826,19 @@ func (s *InferenceService) handleStreamingInference(requestID string, payload In return s.streamFromDockerModel(endpoint, payload.Request, payload.ModelID, localModel, apiKey, sendChunk) } -// resolveModelContexts returns each configured model's real context window in -// tokens: a manual context_length from models.json wins; otherwise the value -// the health checker detected from the backend's /v1/models (max_model_len). -// Models with no known window are omitted — the server falls back to the -// catalog value for those (#61). -func (s *InferenceService) resolveModelContexts() map[string]int { - contexts := make(map[string]int) +// resolveModelContexts returns each configured model's real context window and +// how it was determined: a manual context_length from models.json wins; +// otherwise the value the health checker detected from the backend's +// /v1/models (max_model_len). Models with no known window are omitted — the +// server falls back to the catalog value for those (#61). +// +// The provenance travels with the value because the two paths are not equally +// trustworthy, and the number alone does not say which one produced it. +func (s *InferenceService) resolveModelContexts() map[string]ModelContextInfo { + contexts := make(map[string]ModelContextInfo) for modelID := range s.modelMappings { if info := s.ModelContext(modelID); info.Length > 0 { - contexts[modelID] = info.Length + contexts[modelID] = info } else if info.Source == ContextSourceUnknown { s.warnUnknownContext(modelID) } diff --git a/internal/computing/model_context_source_test.go b/internal/computing/model_context_source_test.go index 3d103db..0fa1665 100644 --- a/internal/computing/model_context_source_test.go +++ b/internal/computing/model_context_source_test.go @@ -65,9 +65,12 @@ func TestResolveOmitsUnknownContext(t *testing.T) { s.healthChecker.recordDetectedContext("org/unknown", nil) got := s.resolveModelContexts() - if len(got) != 1 || got["org/known"] != 8192 { + if len(got) != 1 || got["org/known"].Length != 8192 { t.Fatalf("got %v, want only org/known=8192", got) } + if got["org/known"].Source != ContextSourceDetected { + t.Errorf("source = %q, want %q", got["org/known"].Source, ContextSourceDetected) + } } // The warning is the whole point of the change, but it must not repeat: this diff --git a/internal/computing/model_context_test.go b/internal/computing/model_context_test.go index 1f84b63..c95b201 100644 --- a/internal/computing/model_context_test.go +++ b/internal/computing/model_context_test.go @@ -83,8 +83,10 @@ func TestRecordDetectedContextLocalNameMatch(t *testing.T) { func TestBuildModelMetadataHeartbeatShape(t *testing.T) { c := &InferenceClient{models: []string{"model-a", "model-b", "model-c"}} - c.SetModelContextsProvider(func() map[string]int { - return map[string]int{"model-a": 32768} + c.SetModelContextsProvider(func() map[string]ModelContextInfo { + return map[string]ModelContextInfo{ + "model-a": {Length: 32768, Source: ContextSourceDetected}, + } }) c.SetModelMappingsProvider(func() map[string]ModelMapping { return map[string]ModelMapping{ @@ -126,11 +128,11 @@ func TestResolveModelContextsPrecedence(t *testing.T) { } contexts := s.resolveModelContexts() - if contexts["model-a"] != 32768 { - t.Errorf("expected detected 32768 for model-a, got %d", contexts["model-a"]) + if contexts["model-a"].Length != 32768 || contexts["model-a"].Source != ContextSourceDetected { + t.Errorf("expected detected 32768 for model-a, got %+v", contexts["model-a"]) } - if contexts["model-b"] != 16384 { - t.Errorf("expected manual override 16384 for model-b, got %d", contexts["model-b"]) + if contexts["model-b"].Length != 16384 || contexts["model-b"].Source != ContextSourceOverride { + t.Errorf("expected override 16384 for model-b, got %+v", contexts["model-b"]) } if _, ok := contexts["model-c"]; ok { t.Error("model with unknown context should be omitted")