evals: load eval prompts from disk

scripts/harbor-build-musl-container.sh

@@ -10,6 +10,7 @@ REGISTRY_VOLUME="${BITFUN_HARBOR_MUSL_REGISTRY_VOLUME:-bitfun-harbor-musl-cargo-
 GIT_VOLUME="${BITFUN_HARBOR_MUSL_GIT_VOLUME:-bitfun-harbor-musl-cargo-git}"
 TARGET_TRIPLE="x86_64-unknown-linux-musl"
 BINARY="${ROOT}/target/${TARGET_TRIPLE}/release/bitfun-cli"
+PROMPTS_DIR="${BITFUN_HARBOR_PROMPTS_DIR:-}"
 
 usage() {
   cat <<EOF
@@ -30,12 +31,17 @@ Commands:
 Environment overrides:
   BITFUN_HARBOR_MUSL_IMAGE, BITFUN_HARBOR_MUSL_CONTAINER
   BITFUN_HARBOR_MUSL_REGISTRY_VOLUME, BITFUN_HARBOR_MUSL_GIT_VOLUME
+  BITFUN_HARBOR_PROMPTS_DIR  Optional local prompt directory containing agentic_mode.md
 
 Output binary:
   ${BINARY}
 
-Harbor mount example:
+Harbor mount examples:
   {"type":"bind","source":"${BINARY}","target":"/usr/local/bin/bitfun-cli","read_only":true}
+  {"type":"bind","source":"\${BITFUN_HARBOR_PROMPTS_DIR}","target":"/prompts","read_only":true}
+
+Harbor environment example:
+  BITFUN_PROMPTS_DIR=/prompts
 EOF
 }
 
@@ -62,6 +68,23 @@ docker_exec() {
   fi
 }
 
+prompt_docker_args() {
+  if [[ -z "${PROMPTS_DIR}" ]]; then
+    return 0
+  fi
+  if [[ ! -d "${PROMPTS_DIR}" ]]; then
+    echo "error: prompt directory not found: ${PROMPTS_DIR}" >&2
+    exit 1
+  fi
+  if [[ ! -f "${PROMPTS_DIR}/agentic_mode.md" ]]; then
+    echo "error: prompt directory must contain agentic_mode.md: ${PROMPTS_DIR}" >&2
+    exit 1
+  fi
+  printf '%s\n' \
+    -v "${PROMPTS_DIR}:/prompts:ro" \
+    -e "BITFUN_PROMPTS_DIR=/prompts"
+}
+
 cmd_build_image() {
   docker build -f "${DOCKERFILE}" -t "${IMAGE}" "${ROOT}"
   echo "Built image: ${IMAGE}"
@@ -128,17 +151,24 @@ cmd_test_binary() {
   file "${BINARY}"
   ldd "${BINARY}" || true
 
+  mapfile -t prompt_args < <(prompt_docker_args)
+  if [[ ${#prompt_args[@]} -gt 0 ]]; then
+    echo "Runtime prompts: ${PROMPTS_DIR} -> /prompts"
+  fi
+
   echo
   echo "Ubuntu smoke test:"
   docker run --rm \
     -v "${BINARY}:/usr/local/bin/bitfun-cli:ro" \
+    "${prompt_args[@]}" \
     ubuntu:22.04 \
     /usr/local/bin/bitfun-cli --version
 
   echo
   echo "Alpine smoke test:"
   docker run --rm \
     -v "${BINARY}:/usr/local/bin/bitfun-cli:ro" \
+    "${prompt_args[@]}" \
     alpine:3.20 \
     /usr/local/bin/bitfun-cli --version
 }
@@ -156,6 +186,12 @@ cmd_status() {
   echo "Volumes:"
   echo "  ${REGISTRY_VOLUME}"
   echo "  ${GIT_VOLUME}"
+  echo "Runtime prompts:"
+  if [[ -n "${PROMPTS_DIR}" ]]; then
+    echo "  ${PROMPTS_DIR} -> /prompts (BITFUN_PROMPTS_DIR=/prompts)"
+  else
+    echo "  (not configured; set BITFUN_HARBOR_PROMPTS_DIR)"
+  fi
   echo "Binary:"
   if [[ -e "${BINARY}" ]]; then
     ls -lh "${BINARY}"

src/apps/cli/src/main.rs

@@ -73,6 +73,10 @@ struct Cli {
     /// Disable file logging (stderr logging will still be used)
     #[arg(long, global = true)]
     no_log_file: bool,
+
+    /// Load agent prompt templates from disk instead of only using embedded prompts
+    #[arg(long, global = true, value_name = "DIR")]
+    prompts_dir: Option<PathBuf>,
 }
 
 #[derive(Subcommand)]
@@ -613,6 +617,11 @@ async fn run_cli() -> Result<()> {
         logging::init_file_logging(file_log_level);
     }
 
+    if let Some(prompts_dir) = cli.prompts_dir.as_ref() {
+        bitfun_core::agentic::agents::set_runtime_prompt_dir(prompts_dir);
+        prompts::set_runtime_cli_prompt_dir(prompts_dir);
+    }
+
     let config = CliConfig::load().unwrap_or_else(|e| {
         if !is_tui_mode {
             eprintln!("Warning: Failed to load config: {}", e);

src/apps/cli/src/modes/chat.rs

@@ -1712,14 +1712,9 @@ impl ChatMode {
             "/usage" => {
                 self.show_usage_report(chat_view, chat_state, rt_handle);
             }
-            "/init" => match crate::prompts::get_cli_prompt("init") {
+            "/init" => match crate::prompts::get_cli_prompt_text("init") {
                 Some(prompt) => {
-                    self.send_message_to_agent(
-                        prompt.to_string(),
-                        chat_view,
-                        chat_state,
-                        rt_handle,
-                    );
+                    self.send_message_to_agent(prompt, chat_view, chat_state, rt_handle);
                 }
                 None => {
                     chat_state.add_system_message(

src/apps/cli/src/prompts.rs

@@ -1,3 +1,115 @@
 // Embedded CLI prompts (auto-generated from `prompts/` directory at build time)
 
 include!(concat!(env!("OUT_DIR"), "/embedded_cli_prompts.rs"));
+
+use std::path::{Path, PathBuf};
+use std::sync::OnceLock;
+
+pub const CLI_PROMPTS_ENV: &str = "BITFUN_CLI_PROMPTS_DIR";
+const SHARED_PROMPTS_ENV: &str = "BITFUN_PROMPTS_DIR";
+
+static CLI_PROMPT_DIR_OVERRIDE: OnceLock<PathBuf> = OnceLock::new();
+
+pub fn set_runtime_cli_prompt_dir(path: impl Into<PathBuf>) {
+    let _ = CLI_PROMPT_DIR_OVERRIDE.set(path.into());
+}
+
+pub fn get_cli_prompt_text(name: &str) -> Option<String> {
+    get_runtime_cli_prompt(name).or_else(|| get_cli_prompt(name).map(str::to_string))
+}
+
+fn get_runtime_cli_prompt(name: &str) -> Option<String> {
+    let relative_path = prompt_relative_path(name)?;
+    runtime_prompt_roots()
+        .into_iter()
+        .find_map(|root| read_prompt_file(&root, &relative_path))
+}
+
+fn prompt_relative_path(name: &str) -> Option<PathBuf> {
+    let name = name.trim();
+    if name.is_empty()
+        || name.starts_with('/')
+        || name.starts_with('\\')
+        || name
+            .split('/')
+            .any(|component| component.is_empty() || component == "." || component == "..")
+    {
+        return None;
+    }
+
+    let mut path = PathBuf::new();
+    for component in name.split('/') {
+        path.push(component);
+    }
+    Some(path)
+}
+
+fn runtime_prompt_roots() -> Vec<PathBuf> {
+    let mut roots = Vec::new();
+
+    if let Some(path) = CLI_PROMPT_DIR_OVERRIDE.get() {
+        add_prompt_roots(path, &mut roots);
+    }
+
+    if let Some(path) = std::env::var_os(CLI_PROMPTS_ENV) {
+        add_prompt_roots(Path::new(&path), &mut roots);
+    }
+
+    if let Some(path) = std::env::var_os(SHARED_PROMPTS_ENV) {
+        add_prompt_roots(Path::new(&path), &mut roots);
+    }
+
+    add_prompt_roots(Path::new(env!("CARGO_MANIFEST_DIR")), &mut roots);
+    roots
+}
+
+fn add_prompt_roots(path: &Path, roots: &mut Vec<PathBuf>) {
+    let candidates = [
+        path.to_path_buf(),
+        path.join("prompts"),
+        path.join("src").join("apps").join("cli"),
+        path.join("src").join("apps").join("cli").join("prompts"),
+    ];
+
+    for candidate in candidates {
+        if !candidate.is_dir() {
+            continue;
+        }
+        let normalized = dunce::simplified(&candidate).to_path_buf();
+        if !roots.iter().any(|existing| existing == &normalized) {
+            roots.push(normalized);
+        }
+    }
+}
+
+fn read_prompt_file(root: &Path, relative_path: &Path) -> Option<String> {
+    for extension in ["md", "txt"] {
+        let path = root.join(relative_path).with_extension(extension);
+        if path.is_file() {
+            return std::fs::read_to_string(path).ok();
+        }
+    }
+
+    None
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{prompt_relative_path, read_prompt_file};
+    use std::fs;
+    use std::path::PathBuf;
+
+    #[test]
+    fn runtime_cli_prompt_reads_markdown_file() {
+        let tempdir = tempfile::tempdir().expect("tempdir");
+        fs::write(tempdir.path().join("init.md"), "runtime init").expect("write prompt");
+
+        let prompt = read_prompt_file(tempdir.path(), &PathBuf::from("init")).expect("prompt");
+        assert_eq!(prompt, "runtime init");
+    }
+
+    #[test]
+    fn runtime_cli_prompt_rejects_path_traversal() {
+        assert!(prompt_relative_path("../init").is_none());
+    }
+}

src/apps/cli/src/ui/startup.rs

@@ -1146,10 +1146,10 @@ impl StartupPage {
             "/usage" => {
                 self.status = Some("No active session for /usage.".to_string());
             }
-            "/init" => match crate::prompts::get_cli_prompt("init") {
+            "/init" => match crate::prompts::get_cli_prompt_text("init") {
                 Some(prompt) => {
                     return Some(StartupResult::NewSession {
-                        prompt: Some(prompt.to_string()),
+                        prompt: Some(prompt),
                     });
                 }
                 None => {

src/crates/assembly/core/Cargo.toml

@@ -219,3 +219,6 @@ ssh-remote = [
 
 [build-dependencies]
 sha2 = { workspace = true }
+
+[dev-dependencies]
+tempfile = { workspace = true }

src/crates/assembly/core/src/agentic/agents/definitions/modes/agentic.rs

@@ -7,7 +7,7 @@
 //! only the reminder content differs.
 
 use crate::agentic::agents::{
-    get_embedded_prompt, shared_coding_mode_tool_exposure_overrides, shared_coding_mode_tools,
+    get_prompt_template, shared_coding_mode_tool_exposure_overrides, shared_coding_mode_tools,
     shared_coding_mode_user_context_policy, Agent, AgentToolPolicyOverrides, UserContextPolicy,
     SHARED_CODING_MODE_PROMPT_TEMPLATE,
 };
@@ -38,14 +38,12 @@ impl AgenticMode {
         &self,
         template_name: &str,
     ) -> crate::util::errors::BitFunResult<String> {
-        get_embedded_prompt(template_name)
-            .map(str::to_string)
-            .ok_or_else(|| {
-                crate::util::errors::BitFunError::Agent(format!(
-                    "{} not found in embedded files",
-                    template_name
-                ))
-            })
+        get_prompt_template(template_name).ok_or_else(|| {
+            crate::util::errors::BitFunError::Agent(format!(
+                "{} not found in prompt files",
+                template_name
+            ))
+        })
     }
 }
 

src/crates/assembly/core/src/agentic/agents/definitions/modes/debug.rs

@@ -7,7 +7,7 @@
 //! only the reminder content differs.
 
 use crate::agentic::agents::{
-    get_embedded_prompt, shared_coding_mode_tool_exposure_overrides, shared_coding_mode_tools,
+    get_prompt_template, shared_coding_mode_tool_exposure_overrides, shared_coding_mode_tools,
     shared_coding_mode_user_context_policy, Agent, AgentToolPolicyOverrides, UserContextPolicy,
     SHARED_CODING_MODE_PROMPT_TEMPLATE,
 };
@@ -58,14 +58,12 @@ impl DebugMode {
     }
 
     fn load_reminder_template(&self, template_name: &str) -> BitFunResult<String> {
-        get_embedded_prompt(template_name)
-            .map(str::to_string)
-            .ok_or_else(|| {
-                crate::util::errors::BitFunError::Agent(format!(
-                    "{} not found in embedded files",
-                    template_name
-                ))
-            })
+        get_prompt_template(template_name).ok_or_else(|| {
+            crate::util::errors::BitFunError::Agent(format!(
+                "{} not found in prompt files",
+                template_name
+            ))
+        })
     }
 
     const BUILTIN_JS_TEMPLATE: &'static str = r#"fetch('http://127.0.0.1:{PORT}/ingest/{SESSION_ID}',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({location:'{LOCATION}',message:'{MESSAGE}',data:{DATA},timestamp:Date.now(),sessionId:'{SESSION_ID}',hypothesisId:'{HYPOTHESIS_ID}',runId:'{RUN_ID}'})}).catch(()=>{});"#;

src/crates/assembly/core/src/agentic/agents/definitions/modes/multitask.rs

@@ -7,7 +7,7 @@
 //! only the reminder content differs.
 
 use crate::agentic::agents::{
-    get_embedded_prompt, shared_coding_mode_tool_exposure_overrides, shared_coding_mode_tools,
+    get_prompt_template, shared_coding_mode_tool_exposure_overrides, shared_coding_mode_tools,
     shared_coding_mode_user_context_policy, Agent, AgentToolPolicyOverrides, UserContextPolicy,
     SHARED_CODING_MODE_PROMPT_TEMPLATE,
 };
@@ -39,14 +39,12 @@ impl MultitaskMode {
         &self,
         template_name: &str,
     ) -> crate::util::errors::BitFunResult<String> {
-        get_embedded_prompt(template_name)
-            .map(str::to_string)
-            .ok_or_else(|| {
-                crate::util::errors::BitFunError::Agent(format!(
-                    "{} not found in embedded files",
-                    template_name
-                ))
-            })
+        get_prompt_template(template_name).ok_or_else(|| {
+            crate::util::errors::BitFunError::Agent(format!(
+                "{} not found in prompt files",
+                template_name
+            ))
+        })
     }
 }
 

src/crates/assembly/core/src/agentic/agents/definitions/modes/plan.rs

@@ -7,7 +7,7 @@
 //! only the reminder content differs.
 
 use crate::agentic::agents::{
-    get_embedded_prompt, shared_coding_mode_tool_exposure_overrides, shared_coding_mode_tools,
+    get_prompt_template, shared_coding_mode_tool_exposure_overrides, shared_coding_mode_tools,
     shared_coding_mode_user_context_policy, Agent, AgentToolPolicyOverrides, UserContextPolicy,
     SHARED_CODING_MODE_PROMPT_TEMPLATE,
 };
@@ -39,14 +39,12 @@ impl PlanMode {
         &self,
         template_name: &str,
     ) -> crate::util::errors::BitFunResult<String> {
-        get_embedded_prompt(template_name)
-            .map(str::to_string)
-            .ok_or_else(|| {
-                crate::util::errors::BitFunError::Agent(format!(
-                    "{} not found in embedded files",
-                    template_name
-                ))
-            })
+        get_prompt_template(template_name).ok_or_else(|| {
+            crate::util::errors::BitFunError::Agent(format!(
+                "{} not found in prompt files",
+                template_name
+            ))
+        })
     }
 }