From 0223f66a9ac121cc51ae5a867a7444e76727ba5b Mon Sep 17 00:00:00 2001 From: Alex Fournier Date: Thu, 27 Aug 2026 14:13:33 -0400 Subject: [PATCH 1/2] feat(llm-client): prepare routed completion candidates Signed-off-by: Alex Fournier --- crates/libsy-llm-client/src/lib.rs | 2 +- crates/libsy-llm-client/src/run.rs | 172 +++++++++++++++++++++--- crates/switchyard-runner/src/route.rs | 64 ++------- crates/switchyard-runner/tests/route.rs | 46 +++++-- 4 files changed, 206 insertions(+), 78 deletions(-) diff --git a/crates/libsy-llm-client/src/lib.rs b/crates/libsy-llm-client/src/lib.rs index 059675bbe..f53782555 100644 --- a/crates/libsy-llm-client/src/lib.rs +++ b/crates/libsy-llm-client/src/lib.rs @@ -30,7 +30,7 @@ pub use client::{ModelConfig, TranslatingLlmClient}; pub use error::{LlmClientError, Result}; pub use observation::{LlmCallObservation, RunObservation, RunObserver}; pub use raw::RawResponse; -pub use run::{ClientRouter, run}; +pub use run::{ClientRouter, run, serve_routing_call}; pub use switchyard_translation::RawEventStream; /// Registers process-wide compatibility gauges with the global meter provider. diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rs index df14f22f8..0932d9e77 100644 --- a/crates/libsy-llm-client/src/run.rs +++ b/crates/libsy-llm-client/src/run.rs @@ -8,10 +8,10 @@ //! consumer — it drives the stream with [`switchyard_libsy::drive`], hands routing-time calls to //! a [`RoutedLlmClient`], and serves the terminal routing outcome. //! -//! libsy owns the stream mechanics; what this module adds is ordered candidate fallback and the -//! `libsy.client_call` span around each candidate. Each candidate exhausts its backend retry -//! budget before fallback advances, so the worst case is `candidates × (max_retries + 1)` -//! upstream attempts plus every candidate's backoff. +//! libsy owns the stream mechanics; what this module adds is per-target request preparation, +//! ordered candidate fallback, and the `libsy.client_call` span around each candidate. Each +//! candidate exhausts its backend retry budget before fallback advances, so the worst case is +//! `candidates × (max_retries + 1)` upstream attempts plus every candidate's backoff. use std::collections::HashMap; use std::sync::Arc; @@ -23,6 +23,7 @@ use switchyard_libsy::{Algorithm, CallModel, LibsyError, Result, drive}; use switchyard_protocol::{ LlmClientError, ModelId, Request, Response, RoutedLlmClient, RoutingFallbackReason, }; +use switchyard_translation::prepare_request_for_target; use crate::observation::{LlmCallObservation, RunObservation, RunObserver}; use crate::{metrics, observability}; @@ -84,6 +85,7 @@ pub async fn run( &algorithm_name, &outcome.request, &models, + CallPhase::Completion, &observe, ) .await; @@ -144,22 +146,39 @@ async fn serve( &call.algorithm, &call.request, &call.models, + CallPhase::Routing, &observe, ) .await; call.respond(result) } +/// Serves one model call requested while a libsy algorithm is routing. +/// +/// Candidate failures are returned to the algorithm through the call's response channel. +pub async fn serve_routing_call(clients: ClientRouter, call: CallModel) -> Result<()> { + serve(clients, call, None).await +} + +enum CallPhase { + Routing, + Completion, +} + /// Try candidates in order until one succeeds or a failure stops fallback. async fn call_first_available( clients: &ClientRouter, algorithm: &str, request: &Request, models: &[ModelId], + phase: CallPhase, observe: &(dyn Fn(LlmCallObservation) + Send + Sync), ) -> Result { for (index, target) in models.iter().enumerate() { - let request = request_for(request, target); + let request = match phase { + CallPhase::Routing => clients.prepare_routing_request(request.clone(), target), + CallPhase::Completion => clients.prepare_completion_request(request.clone(), target), + }; match call_one( clients, target, @@ -298,13 +317,6 @@ fn fallback_reason(error: &LibsyError) -> Option { } } -/// Clone a request and stamp the candidate model that should receive it. -fn request_for(request: &Request, target: &ModelId) -> Request { - let mut request = request.clone(); - request.llm_request.model = Some(target.to_string()); - request -} - /// Resolves a routed call's selected model to the client that serves it. /// /// An algorithm routes among named targets; which provider each target lives on is the @@ -315,9 +327,17 @@ fn request_for(request: &Request, target: &ModelId) -> Request { /// Cloning is cheap — the mapping is shared, so one router can serve every request. #[derive(Clone)] pub struct ClientRouter { - routing: Arc, + inner: Arc, } +#[derive(Clone)] +struct ClientRouting { + routing: Routing, + target_prompts: HashMap, + routing_answer_target: Option, +} + +#[derive(Clone)] enum Routing { /// One client serves every model. Single(Arc), @@ -329,7 +349,11 @@ impl ClientRouter { /// Build a router over `model name -> client`, for targets spread across providers. pub fn new(by_model: HashMap>) -> Self { Self { - routing: Arc::new(Routing::ByModel(by_model)), + inner: Arc::new(ClientRouting { + routing: Routing::ByModel(by_model), + target_prompts: HashMap::new(), + routing_answer_target: None, + }), } } @@ -340,10 +364,27 @@ impl ClientRouter { /// only duplicate that. pub fn single(client: Arc) -> Self { Self { - routing: Arc::new(Routing::Single(client)), + inner: Arc::new(ClientRouting { + routing: Routing::Single(client), + target_prompts: HashMap::new(), + routing_answer_target: None, + }), } } + /// Attach system prompts and the target whose routing call may answer the request. + /// Pass `None` when routing only selects a later completion target. + pub fn with_target_prompts( + mut self, + prompts: HashMap, + routing_answer_target: Option, + ) -> Self { + let inner = Arc::make_mut(&mut self.inner); + inner.target_prompts = prompts; + inner.routing_answer_target = routing_answer_target; + self + } + /// The client that serves `model`. /// /// Errors with [`LlmClientError::Configuration`] when the router maps models and has no @@ -352,7 +393,7 @@ impl ClientRouter { &self, model: &ModelId, ) -> std::result::Result<&Arc, LlmClientError> { - match self.routing.as_ref() { + match &self.inner.routing { Routing::Single(client) => Ok(client), Routing::ByModel(by_model) => { by_model @@ -363,6 +404,24 @@ impl ClientRouter { } } } + + /// Prepare a completion candidate with its configured target prompt. + pub fn prepare_completion_request(&self, mut request: Request, target: &ModelId) -> Request { + let prompt = self.inner.target_prompts.get(target).map(String::as_str); + prepare_request_for_target(&mut request.llm_request, target, prompt); + request + } + + /// Prepare a routing call, adding a target prompt only when it generates a candidate answer. + fn prepare_routing_request(&self, mut request: Request, target: &ModelId) -> Request { + let prompt = if self.inner.routing_answer_target.as_ref() == Some(target) { + self.inner.target_prompts.get(target).map(String::as_str) + } else { + None + }; + prepare_request_for_target(&mut request.llm_request, target, prompt); + request + } } impl FromIterator<(ModelId, Arc)> for ClientRouter { @@ -381,8 +440,8 @@ mod tests { use http::StatusCode; use switchyard_libsy::{Driver, RoutingOutcome}; use switchyard_protocol::{ - LlmResponse, LlmResponseChunk, LlmResponseStreamEvent, completion_text, text_request, - text_response, + ContentBlock, LlmResponse, LlmResponseChunk, LlmResponseStreamEvent, completion_text, + text_request, text_response, }; use wiremock::matchers::method; use wiremock::{Mock, MockServer, ResponseTemplate}; @@ -449,6 +508,7 @@ mod tests { struct CandidateClient { calls: Mutex>, + requests: Mutex>, first: FirstOutcome, } @@ -457,6 +517,7 @@ mod tests { async fn call(&self, request: Request) -> std::result::Result { let model = request.model_id().unwrap_or_default(); self.calls.lock().push(model.clone()); + self.requests.lock().push(request); if model == "weak" { return match self.first { FirstOutcome::ContextWindow => Err(LlmClientError::ContextWindowExceeded { @@ -516,11 +577,25 @@ mod tests { } } + fn instruction_text(request: &Request) -> Vec<&str> { + request + .llm_request + .instructions + .iter() + .flat_map(|instruction| &instruction.content) + .filter_map(|block| match block { + ContentBlock::Text { text } => Some(text.as_str()), + _ => None, + }) + .collect() + } + async fn run_candidates( first: FirstOutcome, ) -> (Arc, Result<(ModelId, Response)>) { let client = Arc::new(CandidateClient { calls: Mutex::new(Vec::new()), + requests: Mutex::new(Vec::new()), first, }); let algorithm = Arc::new(CandidateAlgorithm { @@ -540,6 +615,7 @@ mod tests { async fn answered_outcome_does_not_make_a_second_model_call() -> Result<()> { let client = Arc::new(CandidateClient { calls: Mutex::new(Vec::new()), + requests: Mutex::new(Vec::new()), first: FirstOutcome::StreamSuccess, }); let observations = Arc::new(Mutex::new(Vec::new())); @@ -569,6 +645,66 @@ mod tests { Ok(()) } + #[tokio::test] + async fn each_fallback_candidate_receives_only_its_own_prompt() -> Result<()> { + let client = Arc::new(CandidateClient { + calls: Mutex::new(Vec::new()), + requests: Mutex::new(Vec::new()), + first: FirstOutcome::ContextWindow, + }); + let clients = ClientRouter::single(client.clone()).with_target_prompts( + HashMap::from([ + ("weak".into(), "weak prompt".to_string()), + ("strong".into(), "strong prompt".to_string()), + ]), + None, + ); + + run( + Arc::new(CandidateAlgorithm { + models: vec!["weak".into(), "strong".into()], + }), + clients, + request(), + None, + ) + .await?; + + let calls = client.requests.lock(); + assert_eq!(calls.len(), 2); + assert_eq!(instruction_text(&calls[0]), ["weak prompt"]); + assert_eq!(instruction_text(&calls[1]), ["strong prompt"]); + Ok(()) + } + + #[tokio::test] + async fn configured_routing_response_target_receives_its_prompt() -> Result<()> { + let client = Arc::new(CandidateClient { + calls: Mutex::new(Vec::new()), + requests: Mutex::new(Vec::new()), + first: FirstOutcome::StreamSuccess, + }); + let clients = ClientRouter::single(client.clone()).with_target_prompts( + HashMap::from([("answer".into(), "answer prompt".to_string())]), + Some("answer".into()), + ); + + run( + Arc::new(AnsweredAlgorithm { + model: "answer".into(), + }), + clients, + request(), + None, + ) + .await?; + + let calls = client.requests.lock(); + assert_eq!(calls.len(), 1); + assert_eq!(instruction_text(&calls[0]), ["answer prompt"]); + Ok(()) + } + #[test] fn fallback_only_accepts_context_and_unavailable_failures() { let error = |source| LibsyError::client_call("target", source); diff --git a/crates/switchyard-runner/src/route.rs b/crates/switchyard-runner/src/route.rs index dc4db13ee..7a34b9ce5 100644 --- a/crates/switchyard-runner/src/route.rs +++ b/crates/switchyard-runner/src/route.rs @@ -6,9 +6,9 @@ use std::error::Error; use std::sync::Arc; -use libsy::{Algorithm, CallModel, LibsyError, RoutingOutcome, drive}; +use libsy::{Algorithm, LibsyError, RoutingOutcome, drive}; use serde_json::Value; -use switchyard_llm_client::{ClientRouter, RunObserver, TranslatingLlmClient}; +use switchyard_llm_client::{ClientRouter, RunObserver, TranslatingLlmClient, serve_routing_call}; use switchyard_protocol::{LlmClientError, ModelId, Request, Response, WireFormat}; use thiserror::Error; @@ -194,13 +194,17 @@ impl Route { }) } - /// Completes routing-time calls without serving the answer target. + /// Completes routing-time calls and prepares the selected request without serving it. pub async fn decide(&self, request: Request) -> Result { - drive(Arc::clone(&self.algorithm), request, |call| { - serve_decision_dependency(self.clients.clone(), call) + let mut outcome = drive(Arc::clone(&self.algorithm), request, |call| { + serve_routing_call(self.clients.clone(), call) }) .await - .map_err(Into::into) + .map_err(RunnerError::from)?; + outcome.request = self + .clients + .prepare_completion_request(outcome.request, &outcome.selected_model_id); + Ok(outcome) } /// Counts tokens using the configured Anthropic-capable target. @@ -209,6 +213,9 @@ impl Route { .count_tokens_target .as_ref() .ok_or(RunnerError::CountTokensUnsupported)?; + let request = self + .clients + .prepare_completion_request(request, &target.model); target .client .count_tokens(&target.model, request) @@ -216,48 +223,3 @@ impl Route { .map_err(Into::into) } } - -async fn serve_decision_dependency(clients: ClientRouter, call: CallModel) -> libsy::Result<()> { - let mut result = Err(LibsyError::NoTargets); - for (index, model) in call.models.iter().enumerate() { - // The driver stamps only the first candidate, so every fallback must replace it. - let mut request = call.request.clone(); - request.llm_request.model = Some(model.to_string()); - let response = match clients.route(model) { - Ok(client) => client.call(request).await, - Err(source) => Err(source), - }; - match response { - Ok(response) => { - result = Ok(response); - break; - } - Err(source) => { - let try_next = index + 1 < call.models.len() && eligible_routing_fallback(&source); - result = Err(LibsyError::client_call(model.clone(), source)); - if !try_next { - break; - } - } - } - } - call.respond(result) -} - -/// Whether a routing-time candidate failure may fall through to the next model. -fn eligible_routing_fallback(error: &LlmClientError) -> bool { - match error { - LlmClientError::ContextWindowExceeded { .. } - | LlmClientError::Transport { .. } - | LlmClientError::Timeout { .. } => true, - LlmClientError::UpstreamHttp { status, .. } => { - matches!( - *status, - reqwest::StatusCode::FORBIDDEN - | reqwest::StatusCode::REQUEST_TIMEOUT - | reqwest::StatusCode::TOO_MANY_REQUESTS - ) || status.is_server_error() - } - _ => false, - } -} diff --git a/crates/switchyard-runner/tests/route.rs b/crates/switchyard-runner/tests/route.rs index 71ce57c7e..31e5f9bef 100644 --- a/crates/switchyard-runner/tests/route.rs +++ b/crates/switchyard-runner/tests/route.rs @@ -9,8 +9,8 @@ use async_trait::async_trait; use futures_util::StreamExt; use switchyard_llm_client::{ClientRouter, RunObservation}; use switchyard_protocol::{ - LlmClientError, LlmResponse, ModelId, Request, Response, RoutedLlmClient, text_request, - text_response, + ContentBlock, LlmClientError, LlmResponse, ModelId, Request, Response, RoutedLlmClient, + text_request, text_response, }; use switchyard_runner::{AlgorithmSpec, ModelCapabilities, Route}; @@ -29,7 +29,7 @@ impl RoutedLlmClient for StubClient { } } -fn plugin_route(client: Arc) -> Route { +fn plugin_route(client: Arc, target_prompt: Option<&str>) -> Route { let spec = AlgorithmSpec::Passthrough { target: "semantic-target".to_string(), subagents: None, @@ -41,11 +41,19 @@ fn plugin_route(client: Arc) -> Route { let algorithm = spec .build("switchyard", &targets) .expect("identity target map should build"); - let clients = ClientRouter::new( + let mut clients = ClientRouter::new( BTreeMap::from([(ModelId::from("semantic-target"), client)]) .into_iter() .collect(), ); + if let Some(prompt) = target_prompt { + clients = clients.with_target_prompts( + [(ModelId::from("semantic-target"), prompt.to_string())] + .into_iter() + .collect(), + None, + ); + } Route::new( algorithm, clients, @@ -58,7 +66,7 @@ fn plugin_route(client: Arc) -> Route { #[tokio::test] async fn plugin_shaped_route_executes_without_runner_model_or_toml() { - let route = plugin_route(Arc::new(StubClient)); + let route = plugin_route(Arc::new(StubClient), None); let observations = Arc::new(Mutex::new(Vec::new())); let observer = { let observations = Arc::clone(&observations); @@ -101,6 +109,25 @@ async fn plugin_shaped_route_executes_without_runner_model_or_toml() { ); } +#[tokio::test] +async fn decision_prepares_the_selected_target_request() { + let route = plugin_route(Arc::new(StubClient), Some("target prompt")); + let request = Request { + llm_request: text_request(None, "hello"), + ..Request::default() + }; + + let outcome = route + .decide(request) + .await + .expect("passthrough decision should succeed"); + + assert!(matches!( + &outcome.request.llm_request.instructions[0].content[0], + ContentBlock::Text { text } if text == "target prompt" + )); +} + struct LazyStreamClient { polls: Arc, } @@ -124,9 +151,12 @@ impl RoutedLlmClient for LazyStreamClient { #[tokio::test] async fn route_returns_stream_without_polling_it() { let polls = Arc::new(AtomicUsize::new(0)); - let route = plugin_route(Arc::new(LazyStreamClient { - polls: Arc::clone(&polls), - })); + let route = plugin_route( + Arc::new(LazyStreamClient { + polls: Arc::clone(&polls), + }), + None, + ); let request = Request { llm_request: text_request(None, "hello"), ..Request::default() From 03c40160daa217f22af21321448d5e865769dac9 Mon Sep 17 00:00:00 2001 From: Alex Fournier Date: Thu, 27 Aug 2026 14:15:40 -0400 Subject: [PATCH 2/2] feat(server): configure system prompts by target Signed-off-by: Alex Fournier --- crates/switchyard-runner/src/algorithm.rs | 89 ++++++++-- crates/switchyard-runner/src/config.rs | 196 +++++++++++++++++++++- crates/switchyard-server/CONFIGURATION.md | 4 + crates/switchyard-server/README.md | 3 + crates/switchyard-server/tests/server.rs | 93 ++++++++-- docs/reference/toml_schema.md | 12 ++ 6 files changed, 366 insertions(+), 31 deletions(-) diff --git a/crates/switchyard-runner/src/algorithm.rs b/crates/switchyard-runner/src/algorithm.rs index a601ab94a..f70254301 100644 --- a/crates/switchyard-runner/src/algorithm.rs +++ b/crates/switchyard-runner/src/algorithm.rs @@ -513,13 +513,71 @@ impl AlgorithmSpec { } names } + + /// Response target and routing-only dependency for routers that answer while routing. + pub(crate) fn routing_response_and_dependency(&self) -> Option<(&str, &str)> { + match self { + Self::LlmClassifier { config, .. } + if matches!( + config.mode.unwrap_or(if config.escalation.is_some() { + ClassifierMode::Escalation + } else { + ClassifierMode::Capability + }), + ClassifierMode::Escalation + ) => + { + Some(( + config.weak_target.as_deref()?, + config.classifier_target.as_str(), + )) + } + Self::Advisor { + executor_target, + advisor_target, + .. + } => Some((executor_target, advisor_target)), + Self::Noop { .. } + | Self::Random { .. } + | Self::Passthrough { .. } + | Self::LlmClassifier { .. } + | Self::StageRouter { .. } + | Self::Composite { .. } => None, + } + } + + /// Legacy Stage prompt associated with this completion target, when present. + pub(crate) fn legacy_system_prompt(&self, target: &str) -> Option<&str> { + let tiers = match self { + Self::StageRouter { tiers, .. } => Some(tiers), + Self::Composite { stage, .. } => Some(stage), + _ => None, + }; + match tiers { + Some(tiers) if target == tiers.capable_target => tiers.capable_system_prompt.as_deref(), + Some(tiers) if target == tiers.efficient_target => { + tiers.efficient_system_prompt.as_deref() + } + _ => None, + } + } + /// Builds this algorithm after resolving configured target names. pub fn build( &self, context: &str, targets: &BTreeMap, ) -> AlgorithmResult> { - build_algorithm(context, self, targets) + build_algorithm(context, self, targets, true) + } + + /// Builds for the native runner, where the LLM client owns target prompt preparation. + pub(crate) fn build_for_runner( + &self, + context: &str, + targets: &BTreeMap, + ) -> AlgorithmResult> { + build_algorithm(context, self, targets, false) } } impl LlmClassifierRouteConfig { @@ -818,6 +876,7 @@ fn build_algorithm( route_name: &str, config: &AlgorithmSpec, targets: &BTreeMap, + install_legacy_prompts: bool, ) -> AlgorithmResult> { match config { AlgorithmSpec::Noop { .. } => Ok(Arc::new(Noop {})), @@ -958,12 +1017,14 @@ fn build_algorithm( let mut config = StageRouterConfig::new(*picker, *confidence_threshold); config.recent_window = *recent_turn_window; config.handoff_notes = handoff_notes.clone(); - config.tier_prompts = tier_prompts( - &capable, - capable_system_prompt.as_deref(), - &efficient, - efficient_system_prompt.as_deref(), - ); + if install_legacy_prompts { + config.tier_prompts = tier_prompts( + &capable, + capable_system_prompt.as_deref(), + &efficient, + efficient_system_prompt.as_deref(), + ); + } // The judge is called through its own target, so it is not a routing // destination and stays out of the tier pair. config.llm_fallback = classifier @@ -998,12 +1059,14 @@ fn build_algorithm( StageRouterConfig::new(PickerMode::EfficientFirst, stage.confidence_threshold); stage_config.recent_window = stage.recent_turn_window; stage_config.handoff_notes = stage.handoff_notes.clone(); - stage_config.tier_prompts = tier_prompts( - &capable, - stage.capable_system_prompt.as_deref(), - &efficient, - stage.efficient_system_prompt.as_deref(), - ); + if install_legacy_prompts { + stage_config.tier_prompts = tier_prompts( + &capable, + stage.capable_system_prompt.as_deref(), + &efficient, + stage.efficient_system_prompt.as_deref(), + ); + } let config = CompositeRouterConfig { judge_target, judge: classifier.task_classifier_config(), diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs index d4887fab1..e82f5de67 100644 --- a/crates/switchyard-runner/src/config.rs +++ b/crates/switchyard-runner/src/config.rs @@ -184,7 +184,7 @@ impl DeploymentConfig { } let algorithm = config .algorithm - .build(route_name, &targets) + .build_for_runner(route_name, &targets) .map_err(|error| RunnerError::configuration_source(error.to_string(), error))?; let (route_clients, caller_auth) = self.build_route_clients(route_name, config, &clients)?; @@ -288,7 +288,70 @@ impl DeploymentConfig { let client: Arc = client.clone(); by_model.insert(target.id.clone(), client); } - Ok((ClientRouter::new(by_model), caller_auth)) + let (target_prompts, routing_prompt_target) = + self.build_route_target_prompts(route_name, route)?; + let router = + ClientRouter::new(by_model).with_target_prompts(target_prompts, routing_prompt_target); + Ok((router, caller_auth)) + } + + /// Builds the effective prompt policy for this route's completion targets. + fn build_route_target_prompts( + &self, + route_name: &str, + route: &RouteConfig, + ) -> RunnerResult<(HashMap, Option)> { + let mut prompts = HashMap::new(); + let mut aliases = HashMap::<&ModelId, (Option<&str>, bool, bool)>::new(); + for name in route.algorithm.routing_target_names() { + let target = self.targets.get(name).ok_or_else(|| { + RunnerError::configuration(format!("route references unknown target {name}")) + })?; + let legacy_prompt = route.algorithm.legacy_system_prompt(name); + let effective = target.system_prompt.as_deref().or(legacy_prompt); + let (first_prompt, differs, has_target_prompt) = aliases + .entry(&target.id) + .or_insert((effective, false, false)); + *differs |= *first_prompt != effective; + *has_target_prompt |= target.system_prompt.is_some(); + if *differs && *has_target_prompt { + return Err(RunnerError::configuration(format!( + "route {route_name} maps completion target aliases to model {} with different system_prompt values", + target.id + ))); + } + if let Some(prompt) = effective { + prompts.insert(target.id.clone(), prompt.to_string()); + } + } + let routing_prompt_target = if let Some((response_name, dependency_name)) = + route.algorithm.routing_response_and_dependency() + { + let response = self.targets.get(response_name).ok_or_else(|| { + RunnerError::configuration(format!( + "route references unknown target {response_name}" + )) + })?; + if prompts.contains_key(&response.id) { + let dependency = self.targets.get(dependency_name).ok_or_else(|| { + RunnerError::configuration(format!( + "route references unknown target {dependency_name}" + )) + })?; + if response.id == dependency.id { + return Err(RunnerError::configuration(format!( + "route {route_name} cannot apply system_prompt to target {response_name}: model {} is also used by routing-only target {dependency_name}", + response.id, + ))); + } + Some(response.id.clone()) + } else { + None + } + } else { + None + }; + Ok((prompts, routing_prompt_target)) } fn build_count_tokens_target( @@ -379,6 +442,7 @@ struct TargetConfig { llm_client: String, #[serde(default)] extra_body: BTreeMap, + system_prompt: Option, } #[derive(Clone, Copy, Debug, Deserialize)] @@ -533,6 +597,7 @@ bogus = true mod deployment_tests { use super::*; use serde_json::json; + use switchyard_protocol::{ContentBlock, Request, text_request}; const VALID_CONFIG: &str = r#" schema_version = 1 @@ -724,6 +789,133 @@ confidence_threshold = 0.5 Ok(()) } + #[test] + fn target_prompts_override_legacy_stage_and_composite_prompts() -> RunnerResult<()> { + for (route_name, configured) in [ + ( + "stage", + stage_config().replace( + "confidence_threshold = 1.0", + "confidence_threshold = 1.0\ncapable_system_prompt = \"legacy capable\"\nefficient_system_prompt = \"legacy efficient\"", + ), + ), + ( + "composed", + composite_config().replace( + "confidence_threshold = 0.5", + "confidence_threshold = 0.5\ncapable_system_prompt = \"legacy capable\"\nefficient_system_prompt = \"legacy efficient\"", + ), + ), + ] { + let configured = configured.replace( + "id = \"strong/model\"\nllm_client = \"responses\"", + "id = \"strong/model\"\nllm_client = \"responses\"\nsystem_prompt = \"target prompt\"", + ); + let parsed: DeploymentConfig = toml::from_str(&configured).map_err(|error| { + RunnerError::configuration(format!("failed to parse {route_name}: {error}")) + })?; + let route = parsed.routes.get(route_name).ok_or_else(|| { + RunnerError::configuration(format!("route {route_name} is missing")) + })?; + let (prompts, routing_prompt_target) = + parsed.build_route_target_prompts(route_name, route)?; + + assert_eq!( + prompts.get("strong/model").map(String::as_str), + Some("target prompt") + ); + assert_eq!( + prompts.get("weak/model").map(String::as_str), + Some("legacy efficient") + ); + assert!(routing_prompt_target.is_none()); + } + Ok(()) + } + + #[tokio::test] + async fn public_algorithm_build_keeps_legacy_stage_prompting() -> RunnerResult<()> { + let algorithm: AlgorithmSpec = toml::from_str( + r#" +type = "stage_router" +capable_target = "strong" +efficient_target = "weak" +picker = "efficient_first" +confidence_threshold = 1.0 +efficient_system_prompt = "legacy efficient prompt" +"#, + ) + .map_err(|error| RunnerError::configuration(error.to_string()))?; + let targets = BTreeMap::from([ + ("strong".to_string(), ModelId::from("shared/model")), + ("weak".to_string(), ModelId::from("shared/model")), + ]); + let algorithm = algorithm + .build("stage", &targets) + .map_err(|error| RunnerError::configuration(error.to_string()))?; + let request = Request { + llm_request: text_request(None, "routine task"), + ..Request::default() + }; + + let outcome = libsy::drive(algorithm, request, |_| async { + Err(libsy::LibsyError::NoTargets) + }) + .await + .map_err(RunnerError::from)?; + + assert_eq!(outcome.selected_model_id, ModelId::from("shared/model")); + assert!(outcome.request.llm_request.instructions.iter().any(|block| { + block.content.iter().any( + |content| matches!(content, ContentBlock::Text { text } if text == "legacy efficient prompt"), + ) + })); + Ok(()) + } + + #[test] + fn aliased_completion_targets_reject_new_prompt_conflicts() { + let configured = stage_config() + .replace( + "id = \"strong/model\"\nllm_client = \"responses\"", + "id = \"strong/model\"\nllm_client = \"responses\"\nsystem_prompt = \"capable\"", + ) + .replace( + "[routes.stage]", + "[targets.strong_alias]\nid = \"strong/model\"\nllm_client = \"responses\"\n\n[routes.stage]", + ) + .replace("efficient_target = \"weak\"", "efficient_target = \"strong_alias\""); + let message = error_message(&configured); + assert!( + message.contains("completion target aliases to model strong/model with different system_prompt values"), + "unexpected error: {message}" + ); + } + + #[test] + fn prompted_routing_response_cannot_share_a_model_with_a_dependency() { + let configured = VALID_CONFIG + .replace( + "id = \"classifier/model\"\nllm_client = \"primary\"", + "id = \"weak/model\"\nllm_client = \"primary\"", + ) + .replace( + "id = \"weak/model\"\nllm_client = \"anthropic\"", + "id = \"weak/model\"\nllm_client = \"anthropic\"\nsystem_prompt = \"answer prompt\"", + ) + .replace( + "base_threshold = 0.5", + "base_threshold = 0.5\nescalation = { confirmations = 1 }", + ); + + let message = error_message(&configured); + + assert!( + message.contains("cannot apply system_prompt to target weak: model weak/model is also used by routing-only target classifier"), + "unexpected error: {message}" + ); + } + #[test] fn rejects_invalid_unreferenced_llm_client() { let invalid = format!( diff --git a/crates/switchyard-server/CONFIGURATION.md b/crates/switchyard-server/CONFIGURATION.md index ae40d38eb..f7458bc95 100644 --- a/crates/switchyard-server/CONFIGURATION.md +++ b/crates/switchyard-server/CONFIGURATION.md @@ -14,9 +14,13 @@ max_retries = 2 [targets.model] id = "provider/model" llm_client = "provider" +system_prompt = "Follow this model's deployment instructions." extra_body = { chat_template_kwargs = { enable_thinking = false } } ``` +`system_prompt` is prepended when the target is a completion destination. Switchyard +prepares each fallback independently, so a failed target's prompt is not carried to the next one. + `extra_body` is target-specific. It shallow-merges top-level provider options into the outbound request, while explicit request fields win on conflicts. diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.md index 2ff1a7a5a..137dfddde 100644 --- a/crates/switchyard-server/README.md +++ b/crates/switchyard-server/README.md @@ -17,6 +17,7 @@ max_retries = 2 [targets.model_a] id = "model/a" llm_client = "example" +system_prompt = "Use the fast path for routine work." extra_body = { service_tier = "priority" } [targets.model_b] @@ -83,6 +84,8 @@ client's `base_url` should receive the caller's login. A forwarding route must be called through the matching provider API. Target-level `extra_body` values are shallow-merged into the upstream request when the request does not already contain that key. +Target-level `system_prompt` values are prepended when that target serves a completion. +Selected and fallback targets are prepared independently. `max_retries` defaults to `2` and applies to transport failures, timeouts, HTTP 408/429, and 5xx responses. diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index 32290ea36..f478ec92a 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -91,19 +91,36 @@ impl Drop for MockUpstream { } } +fn user_prompt(body: &Value) -> &str { + body["messages"] + .as_array() + .and_then(|messages| messages.iter().find(|message| message["role"] == "user")) + .and_then(|message| message["content"].as_str()) + .unwrap_or_default() +} + +fn has_system_prompt(call: &Value, expected: &str) -> bool { + call["messages"].as_array().is_some_and(|messages| { + messages.iter().any(|message| { + message["role"] == "system" && message["content"].as_str() == Some(expected) + }) + }) +} + async fn upstream_chat( State(calls): State>>>, Json(body): Json, ) -> HttpResponse { calls.lock().await.push(body.clone()); - if body["messages"][0]["content"] == "fail" { + let prompt = user_prompt(&body); + if prompt == "fail" { return ( StatusCode::IM_A_TEAPOT, Json(json!({"error": {"message": "upstream rejected request"}})), ) .into_response(); } - if body["messages"][0]["content"] == "auth-fail" { + if prompt == "auth-fail" { return ( StatusCode::UNAUTHORIZED, Json(json!({"error": {"message": "upstream authentication failed"}})), @@ -112,13 +129,12 @@ async fn upstream_chat( } let model = body["model"].as_str().unwrap_or("unknown").to_string(); - let prompt = body["messages"][0]["content"].as_str().unwrap_or(""); if prompt == "retry-once" && calls .lock() .await .iter() - .filter(|call| call["messages"][0]["content"] == "retry-once") + .filter(|call| user_prompt(call) == "retry-once") .count() == 1 { @@ -136,7 +152,7 @@ async fn upstream_chat( ) .into_response(); } - if model == "model/weak" && body["messages"][0]["content"] == "overflow" { + if model == "model/weak" && prompt == "overflow" { return ( StatusCode::BAD_REQUEST, Json(json!({ @@ -151,7 +167,7 @@ async fn upstream_chat( if body["stream"].as_bool() == Some(true) { // Streamed tool call, for the namespace-on-every-event assertions. The // model calls a tool by the name it was given, so echo that name back. - if body["messages"][0]["content"] == "mcp-tool-call" { + if prompt == "mcp-tool-call" { let called = body["tool_choice"]["function"]["name"] .as_str() .or_else(|| body["tools"][0]["function"]["name"].as_str()) @@ -170,7 +186,7 @@ async fn upstream_chat( ); return Sse::new(stream).into_response(); } - if body["messages"][0]["content"] == "stream-error" { + if prompt == "stream-error" { let events = [ json!({"id": "chatcmpl-stream-error", "model": model, "choices": [{"index": 0, "delta": {"role": "assistant"}}]}).to_string(), json!({"id": "chatcmpl-stream-error", "model": model, "choices": [{"index": 0, "delta": {"content": "before"}}]}).to_string(), @@ -232,7 +248,7 @@ async fn upstream_chat( } // Buffered tool call, the non-streaming counterpart of the branch above. - if body["messages"][0]["content"] == "mcp-tool-call" { + if prompt == "mcp-tool-call" { let called = body["tool_choice"]["function"]["name"] .as_str() .or_else(|| body["tools"][0]["function"]["name"].as_str()) @@ -283,10 +299,9 @@ async fn upstream_chat( } else { r#"{"decision":{"target":"premium"}}"# } - } else if model == "model/classifier" - && body - .pointer("/response_format/json_schema/schema/properties/escalate") - .is_some() + } else if body + .pointer("/response_format/json_schema/schema/properties/escalate") + .is_some() { r#"{"escalate":false,"reason":"making progress"}"# } else if model == "model/classifier" && requests_schema_invalid_verdict { @@ -811,10 +826,12 @@ max_retries = 0 [targets.first] id = "{first}" llm_client = "mock" +system_prompt = "weak answer prompt" [targets.second] id = "{second}" llm_client = "mock" +system_prompt = "strong answer prompt" [routes.random] id = "{ROUTE_MODEL}" @@ -1026,6 +1043,7 @@ base_url = "{model_url}" [targets.judge] id = "model/classifier" llm_client = "judge_provider" +system_prompt = "judge target prompt" [targets.quality] id = "model/strong" @@ -1035,6 +1053,7 @@ llm_client = "model_provider" id = "model/weak" llm_client = "model_provider" extra_body = {{ service_tier = "priority" }} +system_prompt = "economy answer prompt" [routes.classify] id = "switchyard/classify" @@ -1100,6 +1119,10 @@ escalation = {{ confirmations = 1 }} judge_upstream.models().await, vec!["model/classifier".to_string()] ); + assert!(!has_system_prompt( + &judge_upstream.calls.lock().await[0], + "judge target prompt" + )); assert!(model_upstream.models().await.is_empty()); judge_upstream.calls.lock().await.clear(); @@ -1129,6 +1152,14 @@ escalation = {{ confirmations = 1 }} ); assert_eq!(model_upstream.models().await, ["model/weak"]); assert_eq!(judge_upstream.models().await, ["model/classifier"]); + assert!(has_system_prompt( + &model_upstream.calls.lock().await[0], + "economy answer prompt" + )); + assert!(!has_system_prompt( + &judge_upstream.calls.lock().await[0], + "judge target prompt" + )); Ok(()) } @@ -1559,16 +1590,19 @@ format = "openai_chat" base_url = "{base_url}" [targets.classifier] -id = "model/classifier" +id = "model/strong" llm_client = "upstream" +system_prompt = "classifier target prompt" [targets.strong] id = "model/strong" llm_client = "upstream" +system_prompt = "strong answer prompt" [targets.weak] id = "model/weak" llm_client = "upstream" +system_prompt = "weak answer prompt" [routes.escalation] id = "switchyard/escalation" @@ -1596,7 +1630,12 @@ escalation = {{ confirmations = 1 }} ) .await?; assert_eq!(response.status, StatusCode::OK); - assert_eq!(upstream.models().await, ["model/weak", "model/classifier"]); + assert_eq!(upstream.models().await, ["model/weak", "model/strong"]); + let calls = upstream.calls.lock().await; + assert!(has_system_prompt(&calls[0], "weak answer prompt")); + assert!(!has_system_prompt(&calls[1], "classifier target prompt")); + assert!(!has_system_prompt(&calls[1], "strong answer prompt")); + drop(calls); let stats = send( &app, @@ -1611,7 +1650,7 @@ escalation = {{ confirmations = 1 }} assert_eq!(stats["total_prompt_tokens"], 20); assert_eq!(stats["total_completion_tokens"], 4); assert_eq!(stats["models"]["model/weak"]["calls"], 1); - assert_eq!(stats["models"]["model/classifier"]["calls"], 1); + assert_eq!(stats["models"]["model/strong"]["calls"], 1); let process_stats = send(&app, "GET", "/v1/stats", None).await?.json()?; assert_eq!(process_stats["total_requests"], 1); @@ -1726,6 +1765,7 @@ base_url = "{base_url}" [targets.strong] id = "real/opus" llm_client = "claude" +system_prompt = "count target instructions" [targets.other] id = "real/sonnet" @@ -1757,6 +1797,7 @@ targets = ["other", "strong"] assert_eq!(calls.len(), 1); // The inbound route name is rewritten to the real upstream model. assert_eq!(calls[0]["model"], "real/opus"); + assert_eq!(calls[0]["system"], "count target instructions"); Ok(()) } @@ -2447,13 +2488,27 @@ async fn unavailable_target_fails_over_across_endpoints_and_stops_when_exhausted ); assert_eq!(response.json()?["model"], "model/strong"); let calls = upstream.calls.lock().await; + let candidate_calls = &calls[previous_call_count..]; assert_eq!( - calls[previous_call_count..] + candidate_calls .iter() .map(|call| call["model"].as_str().unwrap_or("")) .collect::>(), ["model/weak", "model/strong"] ); + assert!(has_system_prompt(&candidate_calls[0], "weak answer prompt")); + assert!(has_system_prompt( + &candidate_calls[1], + "strong answer prompt" + )); + assert!(!has_system_prompt( + &candidate_calls[0], + "strong answer prompt" + )); + assert!(!has_system_prompt( + &candidate_calls[1], + "weak answer prompt" + )); } let stats = send(&app, "GET", "/v1/stats", None).await?.json()?; @@ -2921,10 +2976,12 @@ base_url = "{base_url}" [targets.executor] id = "model/executor" llm_client = "upstream" +system_prompt = "executor answer prompt" [targets.advisor] id = "model/advisor" llm_client = "upstream" +system_prompt = "advisor target prompt" [routes.gated] id = "switchyard/advisor" @@ -2965,6 +3022,10 @@ async fn advisor_route_approve_flow_and_stats() -> TestResult { ); // Executor turn first, then the review consult. assert_eq!(upstream.models().await, ["model/executor", "model/advisor"]); + let calls = upstream.calls.lock().await; + assert!(has_system_prompt(&calls[0], "executor answer prompt")); + assert!(!has_system_prompt(&calls[1], "advisor target prompt")); + drop(calls); let stats = send(&app, "GET", "/v1/stats", None).await?.json()?; assert_eq!(stats["models"]["model/executor"]["calls"], 1); diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index 6061a67ab..ad30fb586 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -81,8 +81,20 @@ calls an upstream. |---|:---:|---|---| | `id` | Yes | — | Exact model ID sent upstream. | | `llm_client` | Yes | — | Key under `[llm_clients]`. | +| `system_prompt` | No | unset | System prompt prepended when this target serves a completion. | | `extra_body` | No | `{}` | Values merged into the upstream request when the request does not already set that key. | +Each selected or fallback target is prepared from the routed request independently. A prompt +configured for one target is therefore not carried into another target's fallback request. +Judge-only, classifier-only, and reviewer-only targets are not completion destinations and do not +receive this prompt. Existing Stage prompt fields remain supported; `system_prompt` on the target +takes precedence for the same tier. + +Escalation's weak target and Advisor's executor produce a candidate response while routing, so +their target prompt is applied to that call. A prompted target in either role cannot use the same +model ID as that route's judge or reviewer because those calls would otherwise be indistinguishable +at the client boundary; Switchyard rejects that configuration when it loads. + ## `[routes.]` Every route takes the common keys below, plus the keys for its type.