From 0e55b95689a4a79938d38e006e0fe02a6fff475e Mon Sep 17 00:00:00 2001 From: Sylvain Cau Date: Sat, 15 Aug 2026 15:25:21 -0700 Subject: [PATCH 01/11] feat(config)!: layer configuration with figment and nest env vars with __ Configuration was resolved by two overlapping mechanisms: ~40 `Default` impls that read the environment directly, and an `EnvOverride` trait applied after deserialization. Because every struct carries `#[serde(default)]`, the first of those baked the environment into the *defaults* layer, so an environment variable lost to the config file for any section the trait did not cover, and `pdf`, `pdf_handle_cache`, `scheduler`, `email` and `koreader_api` were not covered. Both are replaced by a single figment chain: struct defaults, then the config file, then a sibling `.local` overlay, then the environment. Environment variables now separate nesting levels with `__` and words within a key with `_`, so `rate_limit.anonymous_rps` is `CODEX_RATE_LIMIT__ANONYMOUS_RPS`. The old flat spelling was ambiguous: nothing in `CODEX_RATE_LIMIT_ANONYMOUS_RPS` distinguishes the section `rate_limit` from a section `rate`, which is why the previous implementation needed a hand-written override per key. A `.local.` file next to the config is merged on top when present, so secrets and per-host overrides no longer require editing the committed config. TOML is accepted alongside YAML, chosen by file extension. Rooting the data directories no longer guesses. It used to decide a path was "unset" by comparing it against the literal string `data/`, which silently rewrote the path of anyone who wrote that value deliberately. The loader now builds its layer chain twice, once over the defaults and once without them, and asks the second whether any layer actually supplied the key. Paths keep their `String` type, so no call site changed. Two latent bugs go with it. The environment can now introduce a config subtree the file never mentioned, so switching to `db_type: postgres` by environment alone produces a working section instead of leaving `postgres` as `None` for `display_database_config` to unwrap. And `postgresql`, `http/protobuf` and `http/json` are accepted again as enum spellings, which the old override handled and plain serde did not. Adds tests for the precedence chain, the overlay, path rooting, and a registry-driven test that sets every scalar setting through its `__` name and checks it lands, replacing the per-key tests deleted with the override module. `Config::default()` is now pinned by a snapshot captured before the environment reads were lifted out, so a mistyped literal fails loudly rather than quietly changing a default. `serial_test` is no longer needed; the environment tests run under `figment::Jail`. The startup validator still carries the advisory wording it shipped with, which now reads backwards; switching it to reject the old names is separate. BREAKING CHANGE: every `CODEX_*` configuration environment variable is renamed to use `__` between nesting levels. The old flat names are no longer read. Run `codex config check` on the previous release to get the full list of replacements for the variables you set. --- Cargo.lock | 119 +- crates/codex-cli-common/src/lib.rs | 16 +- crates/codex-config/Cargo.toml | 2 + crates/codex-config/src/env_audit.rs | 4 +- crates/codex-config/src/env_override.rs | 1766 ----------------- crates/codex-config/src/lib.rs | 5 +- crates/codex-config/src/loader.rs | 660 ++++-- .../src/testdata/default_config.json | 138 ++ crates/codex-config/src/types.rs | 558 ++---- src/commands/seed.rs | 5 +- 10 files changed, 977 insertions(+), 2296 deletions(-) delete mode 100644 crates/codex-config/src/env_override.rs create mode 100644 crates/codex-config/src/testdata/default_config.json diff --git a/Cargo.lock b/Cargo.lock index 1910330e5..76c342790 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -286,6 +286,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "atomic" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89cbf775b137e9b968e67227ef7f775587cde3fd31b0d8599dbd0f598a48340" +dependencies = [ + "bytemuck", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -946,6 +955,7 @@ name = "codex-config" version = "1.44.0" dependencies = [ "anyhow", + "figment", "serde", "serde_json", "serde_yaml", @@ -1995,6 +2005,23 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" +[[package]] +name = "figment" +version = "0.10.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cb01cd46b0cf372153850f4c6c272d9cbea2da513e07538405148f95bd789f3" +dependencies = [ + "atomic", + "parking_lot", + "pear", + "serde", + "serde_yaml", + "tempfile", + "toml", + "uncased", + "version_check", +] + [[package]] name = "filetime" version = "0.2.29" @@ -2799,6 +2826,12 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "inlinable_string" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8fae54786f62fb2918dcfae3d568594e50eb9b5c25bf04371af6fe7516452fb" + [[package]] name = "inout" version = "0.1.4" @@ -4043,6 +4076,29 @@ dependencies = [ "web-sys", ] +[[package]] +name = "pear" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdeeaa00ce488657faba8ebf44ab9361f9365a97bd39ffb8a60663f57ff4b467" +dependencies = [ + "inlinable_string", + "pear_codegen", + "yansi", +] + +[[package]] +name = "pear_codegen" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bab5b985dc082b345f812b7df84e1bef27e7207b39e448439ba8bd69c93f147" +dependencies = [ + "proc-macro2", + "proc-macro2-diagnostics", + "quote", + "syn 2.0.117", +] + [[package]] name = "pem" version = "3.0.6" @@ -4304,7 +4360,7 @@ version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "219cb19e96be00ab2e37d6e299658a0cfa83e52429179969b0f0121b4ac46983" dependencies = [ - "toml_edit", + "toml_edit 0.23.10+spec-1.0.0", ] [[package]] @@ -5552,6 +5608,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -6447,6 +6512,27 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime 0.6.11", + "toml_edit 0.22.27", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + [[package]] name = "toml_datetime" version = "0.7.5+spec-1.1.0" @@ -6456,6 +6542,20 @@ dependencies = [ "serde_core", ] +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap 2.13.0", + "serde", + "serde_spanned", + "toml_datetime 0.6.11", + "toml_write", + "winnow", +] + [[package]] name = "toml_edit" version = "0.23.10+spec-1.0.0" @@ -6463,7 +6563,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84c8b9f757e028cee9fa244aea147aab2a9ec09d5325a9b01e0a49730c2b5269" dependencies = [ "indexmap 2.13.0", - "toml_datetime", + "toml_datetime 0.7.5+spec-1.1.0", "toml_parser", "winnow", ] @@ -6477,6 +6577,12 @@ dependencies = [ "winnow", ] +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + [[package]] name = "tonic" version = "0.14.6" @@ -6755,6 +6861,15 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" +[[package]] +name = "uncased" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1b88fcfe09e89d3866a5c11019378088af2d24c3fbd4f0543f96b479ec90697" +dependencies = [ + "version_check", +] + [[package]] name = "unicase" version = "2.9.0" diff --git a/crates/codex-cli-common/src/lib.rs b/crates/codex-cli-common/src/lib.rs index 4f3836ae9..d7b20793f 100644 --- a/crates/codex-cli-common/src/lib.rs +++ b/crates/codex-cli-common/src/lib.rs @@ -1,5 +1,5 @@ use codex_api::observability::ObservabilityHandle; -use codex_config::{Config, DatabaseConfig, DatabaseType, EnvOverride}; +use codex_config::{Config, DatabaseConfig, DatabaseType}; use codex_db::Database; use codex_events::{EventBroadcaster, TaskProgressEvent}; use codex_services::{SettingsService, TaskMetricsService}; @@ -114,19 +114,7 @@ pub fn load_config(config_path: PathBuf) -> anyhow::Result<(Config, bool)> { /// against a read-only mount, and a validation step that writes a config file /// as a side effect would be its own bug. pub fn resolve_config(config_path: &Path) -> anyhow::Result { - let mut config = if config_path.exists() { - Config::from_file(config_path)? - } else { - Config::default() - }; - - // Apply environment variable overrides - config.apply_env_overrides("CODEX"); - - // Resolve sub-directory paths relative to data_dir - config.resolve_data_dir(); - - Ok(config) + Config::load(config_path) } /// Emit a single line when environment variables will need renaming in the diff --git a/crates/codex-config/Cargo.toml b/crates/codex-config/Cargo.toml index e20839eb3..24edd3bbf 100644 --- a/crates/codex-config/Cargo.toml +++ b/crates/codex-config/Cargo.toml @@ -10,10 +10,12 @@ path = "src/lib.rs" [dependencies] serde = { workspace = true } +figment = { version = "0.10", features = ["yaml", "toml", "env"] } serde_json = "1.0" serde_yaml = { workspace = true } anyhow = { workspace = true } [dev-dependencies] +figment = { version = "0.10", features = ["test"] } tempfile = { workspace = true } serial_test = { workspace = true } diff --git a/crates/codex-config/src/env_audit.rs b/crates/codex-config/src/env_audit.rs index f8aa892a4..0a6cfa9d5 100644 --- a/crates/codex-config/src/env_audit.rs +++ b/crates/codex-config/src/env_audit.rs @@ -17,13 +17,11 @@ //! this stays useful long after the v2 rename is behind us. use crate::keys::{KeyRegistry, registry}; +use crate::loader::ENV_PREFIX; use crate::types::Config; use std::collections::{BTreeSet, HashMap}; use std::sync::OnceLock; -/// The prefix every Codex environment variable carries. -pub const ENV_PREFIX: &str = "CODEX_"; - /// `CODEX_`-prefixed variables that are deliberately not config keys. /// /// These are read directly with `std::env::var` at their point of use rather diff --git a/crates/codex-config/src/env_override.rs b/crates/codex-config/src/env_override.rs deleted file mode 100644 index c4fe45dae..000000000 --- a/crates/codex-config/src/env_override.rs +++ /dev/null @@ -1,1766 +0,0 @@ -#[allow(unused_imports)] -use super::types::{ - ApiConfig, ApplicationConfig, AuthConfig, Config, DatabaseConfig, DatabaseType, FilesConfig, - KomgaApiConfig, KoreaderApiConfig, LogLevel, LoggingConfig, ObservabilityBrowserConfig, - ObservabilityConfig, ObservabilityMetricsConfig, ObservabilityTracesConfig, OidcConfig, - OidcDefaultRole, OidcProviderConfig, OtlpConfig, OtlpProtocol, PostgresConfig, RateLimitConfig, - SQLiteConfig, ScannerConfig, TaskConfig, -}; -use std::collections::HashMap; -use std::env; - -/// Trait for applying environment variable overrides to configuration structs -pub trait EnvOverride { - /// Apply environment variable overrides with a given prefix - fn apply_env_overrides(&mut self, prefix: &str); -} - -impl EnvOverride for TaskConfig { - fn apply_env_overrides(&mut self, prefix: &str) { - if let Ok(worker_count) = env::var(format!("{}_WORKER_COUNT", prefix)) - && let Ok(count) = worker_count.parse::() - { - self.worker_count = count; - } - } -} - -impl EnvOverride for ScannerConfig { - fn apply_env_overrides(&mut self, prefix: &str) { - let env_key = format!("{}_MAX_CONCURRENT_SCANS", prefix); - if let Ok(max_scans) = env::var(&env_key) - && let Ok(count) = max_scans.parse::() - { - self.max_concurrent_scans = count; - } - } -} - -impl EnvOverride for KomgaApiConfig { - fn apply_env_overrides(&mut self, prefix: &str) { - if let Ok(enabled) = env::var(format!("{}_ENABLED", prefix)) { - self.enabled = enabled.eq_ignore_ascii_case("true") || enabled == "1"; - } - if let Ok(prefix_value) = env::var(format!("{}_PREFIX", prefix)) - && !prefix_value.is_empty() - { - self.prefix = prefix_value; - } - } -} - -impl EnvOverride for RateLimitConfig { - fn apply_env_overrides(&mut self, prefix: &str) { - if let Ok(enabled) = env::var(format!("{}_ENABLED", prefix)) { - self.enabled = enabled.eq_ignore_ascii_case("true") || enabled == "1"; - } - if let Ok(anonymous_rps) = env::var(format!("{}_ANONYMOUS_RPS", prefix)) - && let Ok(rps) = anonymous_rps.parse::() - { - self.anonymous_rps = rps; - } - if let Ok(anonymous_burst) = env::var(format!("{}_ANONYMOUS_BURST", prefix)) - && let Ok(burst) = anonymous_burst.parse::() - { - self.anonymous_burst = burst; - } - if let Ok(authenticated_rps) = env::var(format!("{}_AUTHENTICATED_RPS", prefix)) - && let Ok(rps) = authenticated_rps.parse::() - { - self.authenticated_rps = rps; - } - if let Ok(authenticated_burst) = env::var(format!("{}_AUTHENTICATED_BURST", prefix)) - && let Ok(burst) = authenticated_burst.parse::() - { - self.authenticated_burst = burst; - } - if let Ok(exempt_paths) = env::var(format!("{}_EXEMPT_PATHS", prefix)) { - self.exempt_paths = exempt_paths - .split(',') - .map(|s| s.trim().to_string()) - .filter(|s| !s.is_empty()) - .collect(); - } - if let Ok(cleanup_interval) = env::var(format!("{}_CLEANUP_INTERVAL_SECS", prefix)) - && let Ok(secs) = cleanup_interval.parse::() - { - self.cleanup_interval_secs = secs; - } - if let Ok(bucket_ttl) = env::var(format!("{}_BUCKET_TTL_SECS", prefix)) - && let Ok(secs) = bucket_ttl.parse::() - { - self.bucket_ttl_secs = secs; - } - } -} - -impl EnvOverride for OidcProviderConfig { - fn apply_env_overrides(&mut self, prefix: &str) { - if let Ok(display_name) = env::var(format!("{}_DISPLAY_NAME", prefix)) { - self.display_name = display_name; - } - if let Ok(issuer_url) = env::var(format!("{}_ISSUER_URL", prefix)) { - self.issuer_url = issuer_url; - } - if let Ok(client_id) = env::var(format!("{}_CLIENT_ID", prefix)) { - self.client_id = client_id; - } - if let Ok(client_secret) = env::var(format!("{}_CLIENT_SECRET", prefix)) { - self.client_secret = Some(client_secret); - } - if let Ok(client_secret_env) = env::var(format!("{}_CLIENT_SECRET_ENV", prefix)) { - self.client_secret_env = Some(client_secret_env); - } - if let Ok(scopes) = env::var(format!("{}_SCOPES", prefix)) { - self.scopes = scopes - .split(',') - .map(|s| s.trim().to_string()) - .filter(|s| !s.is_empty()) - .collect(); - } - if let Ok(groups_claim) = env::var(format!("{}_GROUPS_CLAIM", prefix)) { - self.groups_claim = groups_claim; - } - if let Ok(username_claim) = env::var(format!("{}_USERNAME_CLAIM", prefix)) { - self.username_claim = username_claim; - } - if let Ok(email_claim) = env::var(format!("{}_EMAIL_CLAIM", prefix)) { - self.email_claim = email_claim; - } - if let Ok(accepted_audiences) = env::var(format!("{}_ACCEPTED_AUDIENCES", prefix)) { - self.accepted_audiences = accepted_audiences - .split(',') - .map(|s| s.trim().to_string()) - .filter(|s| !s.is_empty()) - .collect(); - } - - // Role mapping: ROLE_MAPPING_ADMIN="group1, group2", ROLE_MAPPING_MAINTAINER="group3" - let role_mapping_prefix = format!("{}_ROLE_MAPPING_", prefix); - for (key, value) in env::vars() { - if let Some(role_name) = key.strip_prefix(&role_mapping_prefix) { - let role = role_name.to_lowercase(); - let groups: Vec = value - .split(',') - .map(|s| s.trim().to_string()) - .filter(|s| !s.is_empty()) - .collect(); - self.role_mapping.insert(role, groups); - } - } - } -} - -impl EnvOverride for OidcConfig { - fn apply_env_overrides(&mut self, prefix: &str) { - if let Ok(enabled) = env::var(format!("{}_ENABLED", prefix)) { - self.enabled = enabled.eq_ignore_ascii_case("true") || enabled == "1"; - } - if let Ok(auto_create) = env::var(format!("{}_AUTO_CREATE_USERS", prefix)) { - self.auto_create_users = auto_create.eq_ignore_ascii_case("true") || auto_create == "1"; - } - if let Ok(default_role) = env::var(format!("{}_DEFAULT_ROLE", prefix)) { - self.default_role = match default_role.to_lowercase().as_str() { - "admin" => OidcDefaultRole::Admin, - "maintainer" => OidcDefaultRole::Maintainer, - _ => OidcDefaultRole::Reader, - }; - } - if let Ok(redirect_uri_base) = env::var(format!("{}_REDIRECT_URI_BASE", prefix)) { - self.redirect_uri_base = Some(redirect_uri_base); - } - if let Ok(allowed) = env::var(format!("{}_ALLOWED_REDIRECT_URIS", prefix)) { - self.allowed_redirect_uris = parse_csv_list(&allowed); - } - - // Apply overrides to existing providers - for (provider_name, provider_config) in self.providers.iter_mut() { - let provider_prefix = format!( - "{}_PROVIDERS_{}", - prefix, - provider_name.to_uppercase().replace('-', "_") - ); - provider_config.apply_env_overrides(&provider_prefix); - } - - // Check for dynamically configured providers via environment variables - // Format: CODEX_AUTH_OIDC_PROVIDERS__ISSUER_URL (required to detect a new provider) - // This allows adding providers purely through environment variables - for (key, _) in env::vars() { - let providers_prefix = format!("{}_PROVIDERS_", prefix); - if key.starts_with(&providers_prefix) && key.ends_with("_ISSUER_URL") { - // Extract provider name from key - let provider_name_upper = key - .strip_prefix(&providers_prefix) - .and_then(|s| s.strip_suffix("_ISSUER_URL")) - .unwrap_or(""); - if provider_name_upper.is_empty() { - continue; - } - - let provider_name = provider_name_upper.to_lowercase().replace('_', "-"); - let provider_prefix = format!("{}_PROVIDERS_{}", prefix, provider_name_upper); - - // Only create new provider if it doesn't already exist - self.providers - .entry(provider_name.clone()) - .or_insert_with(|| { - // Create provider with defaults and then apply env overrides - let mut new_provider = OidcProviderConfig { - display_name: provider_name, - issuer_url: String::new(), - client_id: String::new(), - client_secret: None, - client_secret_env: None, - scopes: vec!["email".to_string(), "profile".to_string()], - role_mapping: HashMap::new(), - groups_claim: "groups".to_string(), - username_claim: "preferred_username".to_string(), - email_claim: "email".to_string(), - accepted_audiences: vec![], - }; - new_provider.apply_env_overrides(&provider_prefix); - new_provider - }); - } - } - } -} - -impl EnvOverride for Config { - fn apply_env_overrides(&mut self, prefix: &str) { - // Apply data_dir override first (sub-configs may reference it) - if let Ok(data_dir) = env::var(format!("{}_DATA_DIR", prefix)) { - self.data_dir = data_dir; - } - self.application - .apply_env_overrides(&format!("{}_APPLICATION", prefix)); - self.database - .apply_env_overrides(&format!("{}_DATABASE", prefix)); - self.logging - .apply_env_overrides(&format!("{}_LOGGING", prefix)); - self.auth.apply_env_overrides(&format!("{}_AUTH", prefix)); - self.api.apply_env_overrides(&format!("{}_API", prefix)); - self.task.apply_env_overrides(&format!("{}_TASK", prefix)); - self.scanner - .apply_env_overrides(&format!("{}_SCANNER", prefix)); - self.files.apply_env_overrides(&format!("{}_FILES", prefix)); - self.komga_api - .apply_env_overrides(&format!("{}_KOMGA_API", prefix)); - self.rate_limit - .apply_env_overrides(&format!("{}_RATE_LIMIT", prefix)); - self.observability - .apply_env_overrides(&format!("{}_OBSERVABILITY", prefix)); - } -} - -impl EnvOverride for ObservabilityConfig { - fn apply_env_overrides(&mut self, prefix: &str) { - if let Ok(enabled) = env::var(format!("{}_ENABLED", prefix)) { - self.enabled = enabled.eq_ignore_ascii_case("true") || enabled == "1"; - } - if let Ok(service_name) = env::var(format!("{}_SERVICE_NAME", prefix)) - && !service_name.is_empty() - { - self.service_name = service_name; - } - self.otlp.apply_env_overrides(&format!("{}_OTLP", prefix)); - self.traces - .apply_env_overrides(&format!("{}_TRACES", prefix)); - self.metrics - .apply_env_overrides(&format!("{}_METRICS", prefix)); - self.browser - .apply_env_overrides(&format!("{}_BROWSER", prefix)); - } -} - -impl EnvOverride for OtlpConfig { - fn apply_env_overrides(&mut self, prefix: &str) { - if let Ok(endpoint) = env::var(format!("{}_ENDPOINT", prefix)) { - self.endpoint = endpoint; - } - if let Ok(protocol) = env::var(format!("{}_PROTOCOL", prefix)) { - self.protocol = match protocol.to_lowercase().as_str() { - "grpc" => OtlpProtocol::Grpc, - "http/protobuf" | "http-protobuf" | "http_protobuf" | "httpproto" => { - OtlpProtocol::HttpProtobuf - } - "http/json" | "http-json" | "http_json" => OtlpProtocol::HttpJson, - _ => self.protocol, - }; - } - if let Ok(headers) = env::var(format!("{}_HEADERS", prefix)) { - // Format: "k1=v1,k2=v2". Empty pairs are skipped. - self.headers.clear(); - for entry in headers.split(',') { - let entry = entry.trim(); - if entry.is_empty() { - continue; - } - if let Some((k, v)) = entry.split_once('=') { - let k = k.trim(); - let v = v.trim(); - if !k.is_empty() { - self.headers.insert(k.to_string(), v.to_string()); - } - } - } - } - if let Ok(timeout_ms) = env::var(format!("{}_TIMEOUT_MS", prefix)) - && let Ok(ms) = timeout_ms.parse::() - { - self.timeout_ms = ms; - } - if let Ok(proxy_endpoint) = env::var(format!("{}_PROXY_ENDPOINT", prefix)) { - self.proxy_endpoint = if proxy_endpoint.is_empty() { - None - } else { - Some(proxy_endpoint) - }; - } - } -} - -impl EnvOverride for ObservabilityTracesConfig { - fn apply_env_overrides(&mut self, prefix: &str) { - if let Ok(enabled) = env::var(format!("{}_ENABLED", prefix)) { - self.enabled = enabled.eq_ignore_ascii_case("true") || enabled == "1"; - } - if let Ok(sample_ratio) = env::var(format!("{}_SAMPLE_RATIO", prefix)) - && let Ok(ratio) = sample_ratio.parse::() - { - self.sample_ratio = ratio; - } - } -} - -impl EnvOverride for ObservabilityMetricsConfig { - fn apply_env_overrides(&mut self, prefix: &str) { - if let Ok(enabled) = env::var(format!("{}_ENABLED", prefix)) { - self.enabled = enabled.eq_ignore_ascii_case("true") || enabled == "1"; - } - if let Ok(interval_ms) = env::var(format!("{}_EXPORT_INTERVAL_MS", prefix)) - && let Ok(ms) = interval_ms.parse::() - { - self.export_interval_ms = ms; - } - } -} - -impl EnvOverride for ObservabilityBrowserConfig { - fn apply_env_overrides(&mut self, prefix: &str) { - if let Ok(enabled) = env::var(format!("{}_ENABLED", prefix)) { - self.enabled = enabled.eq_ignore_ascii_case("true") || enabled == "1"; - } - if let Ok(proxy_path) = env::var(format!("{}_PROXY_PATH", prefix)) - && !proxy_path.is_empty() - { - self.proxy_path = proxy_path; - } - if let Ok(sample_ratio) = env::var(format!("{}_SAMPLE_RATIO", prefix)) - && let Ok(ratio) = sample_ratio.parse::() - { - self.sample_ratio = ratio; - } - } -} - -impl EnvOverride for ApplicationConfig { - fn apply_env_overrides(&mut self, prefix: &str) { - // Note: application.name moved to database settings - if let Ok(host) = env::var(format!("{}_HOST", prefix)) { - self.host = host; - } - if let Ok(port) = env::var(format!("{}_PORT", prefix)) - && let Ok(port_num) = port.parse() - { - self.port = port_num; - } - if let Ok(base_url) = env::var(format!("{}_BASE_URL", prefix)) { - self.base_url = Some(base_url); - } - } -} - -impl EnvOverride for DatabaseConfig { - fn apply_env_overrides(&mut self, prefix: &str) { - // Check for database type override (db_type in YAML) - if let Ok(db_type) = env::var(format!("{}_DB_TYPE", prefix)) { - if db_type.eq_ignore_ascii_case("postgres") - || db_type.eq_ignore_ascii_case("postgresql") - { - self.db_type = DatabaseType::Postgres; - } else if db_type.eq_ignore_ascii_case("sqlite") { - self.db_type = DatabaseType::SQLite; - } - } - - // Apply PostgreSQL overrides if config exists - if let Some(ref mut pg_config) = self.postgres { - pg_config.apply_env_overrides(&format!("{}_POSTGRES", prefix)); - } - - // Apply SQLite overrides if config exists - if let Some(ref mut sqlite_config) = self.sqlite { - sqlite_config.apply_env_overrides(&format!("{}_SQLITE", prefix)); - } - } -} - -impl EnvOverride for PostgresConfig { - fn apply_env_overrides(&mut self, prefix: &str) { - if let Ok(host) = env::var(format!("{}_HOST", prefix)) { - self.host = host; - } - if let Ok(port) = env::var(format!("{}_PORT", prefix)) - && let Ok(port_num) = port.parse() - { - self.port = port_num; - } - if let Ok(username) = env::var(format!("{}_USERNAME", prefix)) { - self.username = username; - } - if let Ok(password) = env::var(format!("{}_PASSWORD", prefix)) { - self.password = password; - } - if let Ok(database_name) = env::var(format!("{}_DATABASE_NAME", prefix)) { - self.database_name = database_name; - } - } -} - -impl EnvOverride for SQLiteConfig { - fn apply_env_overrides(&mut self, prefix: &str) { - if let Ok(path) = env::var(format!("{}_PATH", prefix)) { - self.path = path; - } - // Note: Pragmas are typically not overridden via env vars due to their complex nature - } -} - -impl EnvOverride for LoggingConfig { - fn apply_env_overrides(&mut self, prefix: &str) { - if let Ok(level_str) = env::var(format!("{}_LEVEL", prefix)) - && let Some(level) = match level_str.to_lowercase().as_str() { - "error" => Some(LogLevel::Error), - "warn" => Some(LogLevel::Warn), - "info" => Some(LogLevel::Info), - "debug" => Some(LogLevel::Debug), - "trace" => Some(LogLevel::Trace), - _ => None, - } - { - self.level = level; - } - - if let Ok(console_str) = env::var(format!("{}_CONSOLE", prefix)) - && let Ok(console_bool) = console_str.parse() - { - self.console = console_bool; - } - - if let Ok(log_file) = env::var(format!("{}_FILE", prefix)) { - self.file = if log_file.is_empty() { - None - } else { - Some(log_file) - }; - } - } -} - -impl EnvOverride for AuthConfig { - fn apply_env_overrides(&mut self, prefix: &str) { - // Check for JWT secret override - print warning if using insecure default - if let Ok(jwt_secret) = env::var(format!("{}_JWT_SECRET", prefix)) { - self.jwt_secret = jwt_secret; - } else if self.jwt_secret == "INSECURE_DEFAULT_SECRET_CHANGE_IN_PRODUCTION" { - eprintln!( - "WARNING: CODEX_AUTH_JWT_SECRET not set, using insecure default for development only!" - ); - } - - if let Ok(jwt_expiry) = env::var(format!("{}_JWT_EXPIRY_HOURS", prefix)) - && let Ok(hours) = jwt_expiry.parse() - { - self.jwt_expiry_hours = hours; - } - if let Ok(refresh_enabled) = env::var(format!("{}_REFRESH_TOKEN_ENABLED", prefix)) { - self.refresh_token_enabled = - refresh_enabled.eq_ignore_ascii_case("true") || refresh_enabled == "1"; - } - if let Ok(refresh_expiry) = env::var(format!("{}_REFRESH_TOKEN_EXPIRY_DAYS", prefix)) - && let Ok(days) = refresh_expiry.parse() - { - self.refresh_token_expiry_days = days; - } - if let Ok(memory_cost) = env::var(format!("{}_ARGON2_MEMORY_COST", prefix)) - && let Ok(cost) = memory_cost.parse() - { - self.argon2_memory_cost = cost; - } - if let Ok(time_cost) = env::var(format!("{}_ARGON2_TIME_COST", prefix)) - && let Ok(cost) = time_cost.parse() - { - self.argon2_time_cost = cost; - } - if let Ok(parallelism) = env::var(format!("{}_ARGON2_PARALLELISM", prefix)) - && let Ok(p) = parallelism.parse() - { - self.argon2_parallelism = p; - } - - // Apply OIDC configuration overrides - self.oidc.apply_env_overrides(&format!("{}_OIDC", prefix)); - } -} - -impl EnvOverride for ApiConfig { - fn apply_env_overrides(&mut self, prefix: &str) { - if let Ok(base_path) = env::var(format!("{}_BASE_PATH", prefix)) { - self.base_path = base_path; - } - if let Ok(enable_api_docs) = env::var(format!("{}_ENABLE_API_DOCS", prefix)) { - self.enable_api_docs = - enable_api_docs.eq_ignore_ascii_case("true") || enable_api_docs == "1"; - } - if let Ok(api_docs_path) = env::var(format!("{}_API_DOCS_PATH", prefix)) { - self.api_docs_path = api_docs_path; - } - if let Ok(cors_enabled) = env::var(format!("{}_CORS_ENABLED", prefix)) { - self.cors_enabled = cors_enabled.eq_ignore_ascii_case("true") || cors_enabled == "1"; - } - if let Ok(cors_origins) = env::var(format!("{}_CORS_ORIGINS", prefix)) { - self.cors_origins = cors_origins - .split(',') - .map(|s| s.trim().to_string()) - .collect(); - } - if let Ok(max_page_size) = env::var(format!("{}_MAX_PAGE_SIZE", prefix)) - && let Ok(size) = max_page_size.parse() - { - self.max_page_size = size; - } - } -} - -impl EnvOverride for FilesConfig { - fn apply_env_overrides(&mut self, prefix: &str) { - if let Ok(thumbnail_dir) = env::var(format!("{}_THUMBNAIL_DIR", prefix)) { - self.thumbnail_dir = thumbnail_dir; - } - if let Ok(uploads_dir) = env::var(format!("{}_UPLOADS_DIR", prefix)) { - self.uploads_dir = uploads_dir; - } - if let Ok(plugins_dir) = env::var(format!("{}_PLUGINS_DIR", prefix)) { - self.plugins_dir = plugins_dir; - } - } -} - -/// Helper function to get environment variable with fallback -pub fn env_or(key: &str, default: T) -> T { - env::var(key) - .ok() - .and_then(|v| v.parse().ok()) - .unwrap_or(default) -} - -/// Helper function to get boolean environment variable with fallback -pub fn env_bool_or(key: &str, default: bool) -> bool { - env::var(key) - .ok() - .map(|v| v.eq_ignore_ascii_case("true") || v == "1") - .unwrap_or(default) -} - -/// Helper function to get optional string environment variable -pub fn env_string_opt(key: &str) -> Option { - env::var(key).ok().filter(|s| !s.is_empty()) -} - -/// Split a comma-separated environment value into trimmed, non-empty entries -pub fn parse_csv_list(value: &str) -> Vec { - value - .split(',') - .map(|entry| entry.trim().to_string()) - .filter(|entry| !entry.is_empty()) - .collect() -} - -#[cfg(test)] -mod tests { - use super::*; - use serial_test::serial; - use std::env; - - // SAFETY: These tests run serially (via #[serial]) so there's no concurrent access to env vars. - // env::set_var and env::remove_var are unsafe in Rust 2024 due to potential data races, - // but serial execution ensures safety here. - fn set_var(key: &str, value: &str) { - unsafe { env::set_var(key, value) } - } - - fn remove_var(key: &str) { - unsafe { env::remove_var(key) } - } - - #[test] - #[serial] - fn test_env_bool_or() { - set_var("TEST_BOOL_TRUE", "true"); - set_var("TEST_BOOL_1", "1"); - set_var("TEST_BOOL_FALSE", "false"); - - assert!(env_bool_or("TEST_BOOL_TRUE", false)); - assert!(env_bool_or("TEST_BOOL_1", false)); - assert!(!env_bool_or("TEST_BOOL_FALSE", false)); - assert!(env_bool_or("NONEXISTENT", true)); - - remove_var("TEST_BOOL_TRUE"); - remove_var("TEST_BOOL_1"); - remove_var("TEST_BOOL_FALSE"); - } - - #[test] - #[serial] - fn test_env_or() { - set_var("TEST_PORT", "9090"); - assert_eq!(env_or("TEST_PORT", 8080u16), 9090); - assert_eq!(env_or("NONEXISTENT", 8080u16), 8080); - remove_var("TEST_PORT"); - } - - #[test] - #[serial] - fn test_env_string_opt() { - set_var("TEST_STRING", "value"); - set_var("TEST_EMPTY", ""); - - assert_eq!(env_string_opt("TEST_STRING"), Some("value".to_string())); - assert_eq!(env_string_opt("TEST_EMPTY"), None); - assert_eq!(env_string_opt("NONEXISTENT"), None); - - remove_var("TEST_STRING"); - remove_var("TEST_EMPTY"); - } - - #[test] - #[serial] - fn test_application_config_override() { - set_var("CODEX_APPLICATION_HOST", "0.0.0.0"); - set_var("CODEX_APPLICATION_PORT", "9090"); - - let mut config = ApplicationConfig { - host: "127.0.0.1".to_string(), - port: 8080, - ..Default::default() - }; - - config.apply_env_overrides("CODEX_APPLICATION"); - - assert_eq!(config.host, "0.0.0.0"); - assert_eq!(config.port, 9090); - - remove_var("CODEX_APPLICATION_HOST"); - remove_var("CODEX_APPLICATION_PORT"); - } - - #[test] - #[serial] - fn test_application_base_url_env_override() { - set_var("CODEX_APPLICATION_BASE_URL", "https://codex.example.com"); - - let mut config = ApplicationConfig { - host: "127.0.0.1".to_string(), - port: 8080, - base_url: None, - }; - - config.apply_env_overrides("CODEX_APPLICATION"); - - assert_eq!( - config.base_url, - Some("https://codex.example.com".to_string()) - ); - assert_eq!(config.effective_base_url(), "https://codex.example.com"); - - remove_var("CODEX_APPLICATION_BASE_URL"); - } - - #[test] - #[serial] - fn test_task_config_env_override() { - // Clear any existing env vars first to avoid interference from other tests - remove_var("CODEX_TASK_WORKER_COUNT"); - - set_var("CODEX_TASK_WORKER_COUNT", "8"); - - let mut config = TaskConfig::default(); - config.apply_env_overrides("CODEX_TASK"); - - assert_eq!(config.worker_count, 8); - - remove_var("CODEX_TASK_WORKER_COUNT"); - } - - #[test] - #[serial] - fn test_task_config_env_override_invalid() { - // Clear any existing env vars first to avoid interference from other tests - remove_var("CODEX_TASK_WORKER_COUNT"); - - set_var("CODEX_TASK_WORKER_COUNT", "invalid"); - - let mut config = TaskConfig { worker_count: 4 }; - config.apply_env_overrides("CODEX_TASK"); - - // Should keep original value if env var is invalid - assert_eq!(config.worker_count, 4); - - remove_var("CODEX_TASK_WORKER_COUNT"); - } - - #[test] - #[serial] - fn test_scanner_config_env_override() { - // Clear any existing env vars first - remove_var("CODEX_SCANNER_MAX_CONCURRENT_SCANS"); - - // Create config with explicit values (not using default which reads env vars) - let mut config = ScannerConfig { - max_concurrent_scans: 2, - }; - - // Set env vars and apply overrides - set_var("CODEX_SCANNER_MAX_CONCURRENT_SCANS", "6"); - config.apply_env_overrides("CODEX_SCANNER"); - - assert_eq!(config.max_concurrent_scans, 6); - - remove_var("CODEX_SCANNER_MAX_CONCURRENT_SCANS"); - } - - #[test] - #[serial] - fn test_scanner_config_env_override_partial() { - // Clear any existing env vars first - remove_var("CODEX_SCANNER_MAX_CONCURRENT_SCANS"); - - // Create config with explicit values (not using default which reads env vars) - let mut config = ScannerConfig { - max_concurrent_scans: 2, - }; - - set_var("CODEX_SCANNER_MAX_CONCURRENT_SCANS", "10"); - config.apply_env_overrides("CODEX_SCANNER"); - - assert_eq!(config.max_concurrent_scans, 10); - - remove_var("CODEX_SCANNER_MAX_CONCURRENT_SCANS"); - } - - #[test] - #[serial] - fn test_config_env_override_task_and_scanner() { - // Clear any existing env vars first - remove_var("CODEX_TASK_WORKER_COUNT"); - remove_var("CODEX_SCANNER_MAX_CONCURRENT_SCANS"); - - // Create config with explicit values to avoid reading env vars in default() - // We'll use a helper to create a minimal config - use crate::{ - ApiConfig, ApplicationConfig, AuthConfig, DatabaseConfig, DatabaseType, EmailConfig, - FilesConfig, KomgaApiConfig, LoggingConfig, ObservabilityConfig, PdfConfig, - PdfHandleCacheConfig, RateLimitConfig, SQLiteConfig, SchedulerConfig, - }; - let mut config = Config { - data_dir: "data".to_string(), - database: DatabaseConfig { - db_type: DatabaseType::SQLite, - postgres: None, - sqlite: Some(SQLiteConfig { - path: "./test.db".to_string(), - pragmas: None, - ..SQLiteConfig::default() - }), - }, - application: ApplicationConfig { - host: "127.0.0.1".to_string(), - port: 8080, - ..Default::default() - }, - logging: LoggingConfig::default(), - auth: AuthConfig::default(), - api: ApiConfig::default(), - email: EmailConfig::default(), - task: TaskConfig { worker_count: 4 }, - scanner: ScannerConfig { - max_concurrent_scans: 2, - }, - scheduler: SchedulerConfig::default(), - files: FilesConfig::default(), - pdf: PdfConfig::default(), - pdf_handle_cache: PdfHandleCacheConfig::default(), - komga_api: KomgaApiConfig::default(), - koreader_api: KoreaderApiConfig::default(), - rate_limit: RateLimitConfig::default(), - observability: ObservabilityConfig::default(), - }; - - // Set env vars BEFORE applying overrides - set_var("CODEX_TASK_WORKER_COUNT", "12"); - set_var("CODEX_SCANNER_MAX_CONCURRENT_SCANS", "5"); - - // Verify env vars are set before applying (capture values for debugging) - let task_var_before = env::var("CODEX_TASK_WORKER_COUNT").ok(); - let scanner_max_var_before = env::var("CODEX_SCANNER_MAX_CONCURRENT_SCANS").ok(); - - // Double-check env vars are still set right before applying (to catch race conditions) - // This ensures we capture the value at the exact moment we need it - let scanner_max_at_call = env::var("CODEX_SCANNER_MAX_CONCURRENT_SCANS").ok(); - - // If env vars are not set at this point, it's a race condition with another test - assert!( - scanner_max_at_call.is_some(), - "Environment variable CODEX_SCANNER_MAX_CONCURRENT_SCANS was cleared by another test (race condition). Value before: {:?}, value at call: {:?}", - scanner_max_var_before, - scanner_max_at_call - ); - - // Apply overrides - Config::apply_env_overrides("CODEX") will call: - // - task.apply_env_overrides("CODEX_TASK") -> looks for CODEX_TASK_WORKER_COUNT - // - scanner.apply_env_overrides("CODEX_SCANNER") -> looks for CODEX_SCANNER_MAX_CONCURRENT_SCANS - // Store value before applying to verify it changes - let scanner_value_before = config.scanner.max_concurrent_scans; - config.apply_env_overrides("CODEX"); - let scanner_value_after = config.scanner.max_concurrent_scans; - - // Debug: Check env var after applying (to catch race conditions) - let scanner_max_var_after = env::var("CODEX_SCANNER_MAX_CONCURRENT_SCANS").ok(); - - // Verify the overrides were applied - assert_eq!( - config.task.worker_count, 12, - "Task worker count should be overridden to 12 (env var before: {:?})", - task_var_before - ); - // Debug: Check what the scanner config looks like after applying - let env_key_used = "CODEX_SCANNER_MAX_CONCURRENT_SCANS".to_string(); - let env_value_when_checked = env::var(&env_key_used).ok(); - - assert_eq!( - scanner_value_after, - 5, - "Scanner max_concurrent_scans should be overridden to 5 (got: {}, was: {}, env var before: {:?}, env var at call: {:?}, env var after: {:?}, env key used: {:?}, env value when checked: {:?})", - scanner_value_after, - scanner_value_before, - scanner_max_var_before, - scanner_max_at_call, - scanner_max_var_after, - env_key_used, - env_value_when_checked - ); - - remove_var("CODEX_TASK_WORKER_COUNT"); - remove_var("CODEX_SCANNER_MAX_CONCURRENT_SCANS"); - } - - #[test] - #[serial] - fn test_komga_api_config_env_override() { - // Clear any existing env vars first - remove_var("CODEX_KOMGA_API_ENABLED"); - remove_var("CODEX_KOMGA_API_PREFIX"); - - // Create config with explicit values - let mut config = KomgaApiConfig { - enabled: false, - prefix: "default".to_string(), - }; - - // Set env vars and apply overrides - set_var("CODEX_KOMGA_API_ENABLED", "true"); - set_var("CODEX_KOMGA_API_PREFIX", "custom"); - config.apply_env_overrides("CODEX_KOMGA_API"); - - assert!(config.enabled); - assert_eq!(config.prefix, "custom"); - - remove_var("CODEX_KOMGA_API_ENABLED"); - remove_var("CODEX_KOMGA_API_PREFIX"); - } - - #[test] - #[serial] - fn test_komga_api_config_env_override_enabled_with_1() { - // Test that "1" is also accepted for enabled - remove_var("CODEX_KOMGA_API_ENABLED"); - - let mut config = KomgaApiConfig { - enabled: false, - prefix: "default".to_string(), - }; - - set_var("CODEX_KOMGA_API_ENABLED", "1"); - config.apply_env_overrides("CODEX_KOMGA_API"); - - assert!(config.enabled); - - remove_var("CODEX_KOMGA_API_ENABLED"); - } - - #[test] - #[serial] - fn test_komga_api_config_env_override_partial() { - // Test that partial env vars work (only enabled, not prefix) - remove_var("CODEX_KOMGA_API_ENABLED"); - remove_var("CODEX_KOMGA_API_PREFIX"); - - let mut config = KomgaApiConfig { - enabled: false, - prefix: "original".to_string(), - }; - - set_var("CODEX_KOMGA_API_ENABLED", "true"); - // Don't set PREFIX - config.apply_env_overrides("CODEX_KOMGA_API"); - - assert!(config.enabled); - assert_eq!(config.prefix, "original"); // Should remain unchanged - - remove_var("CODEX_KOMGA_API_ENABLED"); - } - - #[test] - #[serial] - fn test_komga_api_config_env_override_empty_prefix_ignored() { - // Test that empty PREFIX env var is ignored - remove_var("CODEX_KOMGA_API_PREFIX"); - - let mut config = KomgaApiConfig { - enabled: false, - prefix: "original".to_string(), - }; - - set_var("CODEX_KOMGA_API_PREFIX", ""); - config.apply_env_overrides("CODEX_KOMGA_API"); - - assert_eq!(config.prefix, "original"); // Should remain unchanged - - remove_var("CODEX_KOMGA_API_PREFIX"); - } - - #[test] - #[serial] - fn test_config_komga_api_env_override_via_main_config() { - // Test that komga_api env overrides work through Config::apply_env_overrides - remove_var("CODEX_KOMGA_API_ENABLED"); - remove_var("CODEX_KOMGA_API_PREFIX"); - - use crate::{ - ApiConfig, ApplicationConfig, AuthConfig, DatabaseConfig, DatabaseType, EmailConfig, - FilesConfig, KomgaApiConfig, LoggingConfig, ObservabilityConfig, PdfConfig, - PdfHandleCacheConfig, RateLimitConfig, SQLiteConfig, SchedulerConfig, - }; - let mut config = Config { - data_dir: "data".to_string(), - database: DatabaseConfig { - db_type: DatabaseType::SQLite, - postgres: None, - sqlite: Some(SQLiteConfig { - path: "./test.db".to_string(), - pragmas: None, - ..SQLiteConfig::default() - }), - }, - application: ApplicationConfig { - host: "127.0.0.1".to_string(), - port: 8080, - ..Default::default() - }, - logging: LoggingConfig::default(), - auth: AuthConfig::default(), - api: ApiConfig::default(), - email: EmailConfig::default(), - task: TaskConfig { worker_count: 4 }, - scanner: ScannerConfig { - max_concurrent_scans: 2, - }, - scheduler: SchedulerConfig::default(), - files: FilesConfig::default(), - pdf: PdfConfig::default(), - pdf_handle_cache: PdfHandleCacheConfig::default(), - komga_api: KomgaApiConfig { - enabled: false, - prefix: "default".to_string(), - }, - koreader_api: KoreaderApiConfig::default(), - rate_limit: RateLimitConfig::default(), - observability: ObservabilityConfig::default(), - }; - - set_var("CODEX_KOMGA_API_ENABLED", "true"); - set_var("CODEX_KOMGA_API_PREFIX", "mykomga"); - config.apply_env_overrides("CODEX"); - - assert!(config.komga_api.enabled); - assert_eq!(config.komga_api.prefix, "mykomga"); - - remove_var("CODEX_KOMGA_API_ENABLED"); - remove_var("CODEX_KOMGA_API_PREFIX"); - } - - #[test] - #[serial] - fn test_rate_limit_config_env_override() { - // Clear any existing env vars first - remove_var("CODEX_RATE_LIMIT_ENABLED"); - remove_var("CODEX_RATE_LIMIT_ANONYMOUS_RPS"); - remove_var("CODEX_RATE_LIMIT_ANONYMOUS_BURST"); - remove_var("CODEX_RATE_LIMIT_AUTHENTICATED_RPS"); - remove_var("CODEX_RATE_LIMIT_AUTHENTICATED_BURST"); - remove_var("CODEX_RATE_LIMIT_EXEMPT_PATHS"); - remove_var("CODEX_RATE_LIMIT_CLEANUP_INTERVAL_SECS"); - remove_var("CODEX_RATE_LIMIT_BUCKET_TTL_SECS"); - - // Create config with explicit values - let mut config = RateLimitConfig { - enabled: true, - anonymous_rps: 10, - anonymous_burst: 50, - authenticated_rps: 50, - authenticated_burst: 200, - exempt_paths: vec!["/health".to_string()], - cleanup_interval_secs: 60, - bucket_ttl_secs: 300, - }; - - // Set env vars and apply overrides - set_var("CODEX_RATE_LIMIT_ENABLED", "false"); - set_var("CODEX_RATE_LIMIT_ANONYMOUS_RPS", "20"); - set_var("CODEX_RATE_LIMIT_ANONYMOUS_BURST", "100"); - set_var("CODEX_RATE_LIMIT_AUTHENTICATED_RPS", "100"); - set_var("CODEX_RATE_LIMIT_AUTHENTICATED_BURST", "400"); - set_var("CODEX_RATE_LIMIT_EXEMPT_PATHS", "/health, /metrics"); - set_var("CODEX_RATE_LIMIT_CLEANUP_INTERVAL_SECS", "120"); - set_var("CODEX_RATE_LIMIT_BUCKET_TTL_SECS", "600"); - config.apply_env_overrides("CODEX_RATE_LIMIT"); - - assert!(!config.enabled); - assert_eq!(config.anonymous_rps, 20); - assert_eq!(config.anonymous_burst, 100); - assert_eq!(config.authenticated_rps, 100); - assert_eq!(config.authenticated_burst, 400); - assert_eq!( - config.exempt_paths, - vec!["/health".to_string(), "/metrics".to_string()] - ); - assert_eq!(config.cleanup_interval_secs, 120); - assert_eq!(config.bucket_ttl_secs, 600); - - remove_var("CODEX_RATE_LIMIT_ENABLED"); - remove_var("CODEX_RATE_LIMIT_ANONYMOUS_RPS"); - remove_var("CODEX_RATE_LIMIT_ANONYMOUS_BURST"); - remove_var("CODEX_RATE_LIMIT_AUTHENTICATED_RPS"); - remove_var("CODEX_RATE_LIMIT_AUTHENTICATED_BURST"); - remove_var("CODEX_RATE_LIMIT_EXEMPT_PATHS"); - remove_var("CODEX_RATE_LIMIT_CLEANUP_INTERVAL_SECS"); - remove_var("CODEX_RATE_LIMIT_BUCKET_TTL_SECS"); - } - - #[test] - #[serial] - fn test_rate_limit_config_env_override_enabled_with_1() { - // Test that "1" is also accepted for enabled - remove_var("CODEX_RATE_LIMIT_ENABLED"); - - let mut config = RateLimitConfig { - enabled: false, - anonymous_rps: 10, - anonymous_burst: 50, - authenticated_rps: 50, - authenticated_burst: 200, - exempt_paths: vec![], - cleanup_interval_secs: 60, - bucket_ttl_secs: 300, - }; - - set_var("CODEX_RATE_LIMIT_ENABLED", "1"); - config.apply_env_overrides("CODEX_RATE_LIMIT"); - - assert!(config.enabled); - - remove_var("CODEX_RATE_LIMIT_ENABLED"); - } - - #[test] - #[serial] - fn test_rate_limit_config_env_override_partial() { - // Test that partial env vars work (only enabled and anonymous_rps) - remove_var("CODEX_RATE_LIMIT_ENABLED"); - remove_var("CODEX_RATE_LIMIT_ANONYMOUS_RPS"); - - let mut config = RateLimitConfig { - enabled: true, - anonymous_rps: 10, - anonymous_burst: 50, - authenticated_rps: 50, - authenticated_burst: 200, - exempt_paths: vec!["/original".to_string()], - cleanup_interval_secs: 60, - bucket_ttl_secs: 300, - }; - - set_var("CODEX_RATE_LIMIT_ENABLED", "false"); - set_var("CODEX_RATE_LIMIT_ANONYMOUS_RPS", "5"); - config.apply_env_overrides("CODEX_RATE_LIMIT"); - - assert!(!config.enabled); - assert_eq!(config.anonymous_rps, 5); - // These should remain unchanged - assert_eq!(config.anonymous_burst, 50); - assert_eq!(config.authenticated_rps, 50); - assert_eq!(config.exempt_paths, vec!["/original".to_string()]); - - remove_var("CODEX_RATE_LIMIT_ENABLED"); - remove_var("CODEX_RATE_LIMIT_ANONYMOUS_RPS"); - } - - #[test] - #[serial] - fn test_rate_limit_config_env_override_invalid_values_ignored() { - // Test that invalid env var values are ignored - remove_var("CODEX_RATE_LIMIT_ANONYMOUS_RPS"); - remove_var("CODEX_RATE_LIMIT_CLEANUP_INTERVAL_SECS"); - - let mut config = RateLimitConfig { - enabled: true, - anonymous_rps: 10, - anonymous_burst: 50, - authenticated_rps: 50, - authenticated_burst: 200, - exempt_paths: vec![], - cleanup_interval_secs: 60, - bucket_ttl_secs: 300, - }; - - set_var("CODEX_RATE_LIMIT_ANONYMOUS_RPS", "invalid"); - set_var("CODEX_RATE_LIMIT_CLEANUP_INTERVAL_SECS", "not_a_number"); - config.apply_env_overrides("CODEX_RATE_LIMIT"); - - // Should keep original values when env vars are invalid - assert_eq!(config.anonymous_rps, 10); - assert_eq!(config.cleanup_interval_secs, 60); - - remove_var("CODEX_RATE_LIMIT_ANONYMOUS_RPS"); - remove_var("CODEX_RATE_LIMIT_CLEANUP_INTERVAL_SECS"); - } - - // OIDC Configuration Environment Override Tests - - #[test] - #[serial] - fn test_oidc_config_env_override() { - // Clear any existing env vars first - remove_var("CODEX_AUTH_OIDC_ENABLED"); - remove_var("CODEX_AUTH_OIDC_AUTO_CREATE_USERS"); - remove_var("CODEX_AUTH_OIDC_DEFAULT_ROLE"); - - use crate::{OidcConfig, OidcDefaultRole}; - - let mut config = OidcConfig { - enabled: false, - auto_create_users: true, - default_role: OidcDefaultRole::Reader, - redirect_uri_base: None, - allowed_redirect_uris: vec![], - providers: std::collections::HashMap::new(), - }; - - // Set env vars and apply overrides - set_var("CODEX_AUTH_OIDC_ENABLED", "true"); - set_var("CODEX_AUTH_OIDC_AUTO_CREATE_USERS", "false"); - set_var("CODEX_AUTH_OIDC_DEFAULT_ROLE", "admin"); - config.apply_env_overrides("CODEX_AUTH_OIDC"); - - assert!(config.enabled); - assert!(!config.auto_create_users); - assert!(matches!(config.default_role, OidcDefaultRole::Admin)); - - remove_var("CODEX_AUTH_OIDC_ENABLED"); - remove_var("CODEX_AUTH_OIDC_AUTO_CREATE_USERS"); - remove_var("CODEX_AUTH_OIDC_DEFAULT_ROLE"); - } - - #[test] - #[serial] - fn test_oidc_config_env_override_allowed_redirect_uris() { - use crate::{OidcConfig, OidcDefaultRole}; - - remove_var("CODEX_AUTH_OIDC_ALLOWED_REDIRECT_URIS"); - - let mut config = OidcConfig { - enabled: true, - auto_create_users: true, - default_role: OidcDefaultRole::Reader, - redirect_uri_base: None, - allowed_redirect_uris: vec![], - providers: std::collections::HashMap::new(), - }; - - // No env var set leaves the allowlist empty, which permits no redirect target - config.apply_env_overrides("CODEX_AUTH_OIDC"); - assert!(config.allowed_redirect_uris.is_empty()); - - set_var( - "CODEX_AUTH_OIDC_ALLOWED_REDIRECT_URIS", - "codexreader://auth, https://app.example.com/callback ,", - ); - config.apply_env_overrides("CODEX_AUTH_OIDC"); - - assert_eq!( - config.allowed_redirect_uris, - vec![ - "codexreader://auth".to_string(), - "https://app.example.com/callback".to_string(), - ], - "entries should be trimmed and empty ones dropped" - ); - - remove_var("CODEX_AUTH_OIDC_ALLOWED_REDIRECT_URIS"); - } - - #[test] - #[serial] - fn test_oidc_config_env_override_enabled_with_1() { - remove_var("CODEX_AUTH_OIDC_ENABLED"); - - use crate::{OidcConfig, OidcDefaultRole}; - - let mut config = OidcConfig { - enabled: false, - auto_create_users: true, - default_role: OidcDefaultRole::Reader, - redirect_uri_base: None, - allowed_redirect_uris: vec![], - providers: std::collections::HashMap::new(), - }; - - set_var("CODEX_AUTH_OIDC_ENABLED", "1"); - config.apply_env_overrides("CODEX_AUTH_OIDC"); - - assert!(config.enabled); - - remove_var("CODEX_AUTH_OIDC_ENABLED"); - } - - #[test] - #[serial] - fn test_oidc_config_env_override_default_role_variants() { - use crate::{OidcConfig, OidcDefaultRole}; - - // Test maintainer role - remove_var("CODEX_AUTH_OIDC_DEFAULT_ROLE"); - - let mut config = OidcConfig { - enabled: false, - auto_create_users: true, - default_role: OidcDefaultRole::Reader, - redirect_uri_base: None, - allowed_redirect_uris: vec![], - providers: std::collections::HashMap::new(), - }; - - set_var("CODEX_AUTH_OIDC_DEFAULT_ROLE", "maintainer"); - config.apply_env_overrides("CODEX_AUTH_OIDC"); - assert!(matches!(config.default_role, OidcDefaultRole::Maintainer)); - - // Test reader role (explicit) - set_var("CODEX_AUTH_OIDC_DEFAULT_ROLE", "reader"); - config.apply_env_overrides("CODEX_AUTH_OIDC"); - assert!(matches!(config.default_role, OidcDefaultRole::Reader)); - - remove_var("CODEX_AUTH_OIDC_DEFAULT_ROLE"); - } - - #[test] - #[serial] - fn test_oidc_provider_config_env_override() { - remove_var("CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_DISPLAY_NAME"); - remove_var("CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_ISSUER_URL"); - remove_var("CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_CLIENT_ID"); - remove_var("CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_CLIENT_SECRET"); - remove_var("CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_SCOPES"); - remove_var("CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_GROUPS_CLAIM"); - remove_var("CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_ACCEPTED_AUDIENCES"); - - use crate::OidcProviderConfig; - - let mut provider = OidcProviderConfig { - display_name: "Original".to_string(), - issuer_url: "https://original.example.com".to_string(), - client_id: "original-client".to_string(), - client_secret: None, - client_secret_env: None, - scopes: vec![], - role_mapping: std::collections::HashMap::new(), - groups_claim: "groups".to_string(), - username_claim: "preferred_username".to_string(), - email_claim: "email".to_string(), - accepted_audiences: vec![], - }; - - set_var( - "CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_DISPLAY_NAME", - "Authentik SSO", - ); - set_var( - "CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_ISSUER_URL", - "https://auth.example.com", - ); - set_var( - "CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_CLIENT_ID", - "new-client-id", - ); - set_var( - "CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_CLIENT_SECRET", - "secret123", - ); - set_var( - "CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_SCOPES", - "email, profile, groups", - ); - set_var( - "CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_GROUPS_CLAIM", - "custom_groups", - ); - set_var( - "CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_ACCEPTED_AUDIENCES", - "codex-client, shared-shisho-client", - ); - provider.apply_env_overrides("CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK"); - - assert_eq!(provider.display_name, "Authentik SSO"); - assert_eq!(provider.issuer_url, "https://auth.example.com"); - assert_eq!(provider.client_id, "new-client-id"); - assert_eq!(provider.client_secret, Some("secret123".to_string())); - assert_eq!( - provider.scopes, - vec![ - "email".to_string(), - "profile".to_string(), - "groups".to_string() - ] - ); - assert_eq!(provider.groups_claim, "custom_groups"); - assert_eq!( - provider.accepted_audiences, - vec![ - "codex-client".to_string(), - "shared-shisho-client".to_string() - ] - ); - - remove_var("CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_DISPLAY_NAME"); - remove_var("CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_ISSUER_URL"); - remove_var("CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_CLIENT_ID"); - remove_var("CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_CLIENT_SECRET"); - remove_var("CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_SCOPES"); - remove_var("CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_GROUPS_CLAIM"); - remove_var("CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_ACCEPTED_AUDIENCES"); - } - - #[test] - #[serial] - fn test_oidc_config_existing_provider_env_override() { - // Test that env vars override existing provider config - remove_var("CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_CLIENT_ID"); - remove_var("CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_CLIENT_SECRET"); - - use crate::{OidcConfig, OidcDefaultRole, OidcProviderConfig}; - - let mut providers = std::collections::HashMap::new(); - providers.insert( - "authentik".to_string(), - OidcProviderConfig { - display_name: "Authentik".to_string(), - issuer_url: "https://auth.example.com".to_string(), - client_id: "yaml-client".to_string(), - client_secret: Some("yaml-secret".to_string()), - client_secret_env: None, - scopes: vec!["email".to_string()], - role_mapping: std::collections::HashMap::new(), - groups_claim: "groups".to_string(), - username_claim: "preferred_username".to_string(), - email_claim: "email".to_string(), - accepted_audiences: vec![], - }, - ); - - let mut config = OidcConfig { - enabled: true, - auto_create_users: true, - default_role: OidcDefaultRole::Reader, - redirect_uri_base: None, - allowed_redirect_uris: vec![], - providers, - }; - - // Override client_id and client_secret via env vars - set_var( - "CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_CLIENT_ID", - "env-client", - ); - set_var( - "CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_CLIENT_SECRET", - "env-secret", - ); - config.apply_env_overrides("CODEX_AUTH_OIDC"); - - let provider = config.providers.get("authentik").unwrap(); - assert_eq!(provider.client_id, "env-client"); - assert_eq!(provider.client_secret, Some("env-secret".to_string())); - // Non-overridden values should remain - assert_eq!(provider.display_name, "Authentik"); - assert_eq!(provider.issuer_url, "https://auth.example.com"); - - remove_var("CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_CLIENT_ID"); - remove_var("CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_CLIENT_SECRET"); - } - - #[test] - #[serial] - fn test_oidc_config_dynamic_provider_creation_via_env() { - // Test that a new provider can be created purely through env vars - // This requires ISSUER_URL to be set (used as detection mechanism) - remove_var("CODEX_AUTH_OIDC_PROVIDERS_NEWPROVIDER_ISSUER_URL"); - remove_var("CODEX_AUTH_OIDC_PROVIDERS_NEWPROVIDER_CLIENT_ID"); - remove_var("CODEX_AUTH_OIDC_PROVIDERS_NEWPROVIDER_CLIENT_SECRET"); - remove_var("CODEX_AUTH_OIDC_PROVIDERS_NEWPROVIDER_DISPLAY_NAME"); - - use crate::{OidcConfig, OidcDefaultRole}; - - let mut config = OidcConfig { - enabled: true, - auto_create_users: true, - default_role: OidcDefaultRole::Reader, - redirect_uri_base: None, - allowed_redirect_uris: vec![], - providers: std::collections::HashMap::new(), - }; - - // Set env vars to create a new provider - set_var( - "CODEX_AUTH_OIDC_PROVIDERS_NEWPROVIDER_ISSUER_URL", - "https://new.example.com", - ); - set_var( - "CODEX_AUTH_OIDC_PROVIDERS_NEWPROVIDER_CLIENT_ID", - "new-client", - ); - set_var( - "CODEX_AUTH_OIDC_PROVIDERS_NEWPROVIDER_CLIENT_SECRET", - "new-secret", - ); - set_var( - "CODEX_AUTH_OIDC_PROVIDERS_NEWPROVIDER_DISPLAY_NAME", - "New Provider", - ); - config.apply_env_overrides("CODEX_AUTH_OIDC"); - - // The provider should now exist (key is lowercase with hyphens) - assert!(config.providers.contains_key("newprovider")); - let provider = config.providers.get("newprovider").unwrap(); - assert_eq!(provider.issuer_url, "https://new.example.com"); - assert_eq!(provider.client_id, "new-client"); - assert_eq!(provider.client_secret, Some("new-secret".to_string())); - assert_eq!(provider.display_name, "New Provider"); - - remove_var("CODEX_AUTH_OIDC_PROVIDERS_NEWPROVIDER_ISSUER_URL"); - remove_var("CODEX_AUTH_OIDC_PROVIDERS_NEWPROVIDER_CLIENT_ID"); - remove_var("CODEX_AUTH_OIDC_PROVIDERS_NEWPROVIDER_CLIENT_SECRET"); - remove_var("CODEX_AUTH_OIDC_PROVIDERS_NEWPROVIDER_DISPLAY_NAME"); - } - - #[test] - #[serial] - fn test_auth_config_oidc_env_override_via_parent() { - // Test that OIDC env overrides work through AuthConfig::apply_env_overrides - remove_var("CODEX_AUTH_OIDC_ENABLED"); - remove_var("CODEX_AUTH_OIDC_AUTO_CREATE_USERS"); - - use crate::{AuthConfig, OidcConfig, OidcDefaultRole}; - - let mut config = AuthConfig { - jwt_secret: "test-secret".to_string(), - jwt_expiry_hours: 24, - refresh_token_enabled: false, - refresh_token_expiry_days: 30, - email_confirmation_required: false, - argon2_memory_cost: 19456, - argon2_time_cost: 2, - argon2_parallelism: 1, - oidc: OidcConfig { - enabled: false, - auto_create_users: true, - default_role: OidcDefaultRole::Reader, - redirect_uri_base: None, - allowed_redirect_uris: vec![], - providers: std::collections::HashMap::new(), - }, - }; - - set_var("CODEX_AUTH_OIDC_ENABLED", "true"); - set_var("CODEX_AUTH_OIDC_AUTO_CREATE_USERS", "false"); - config.apply_env_overrides("CODEX_AUTH"); - - assert!(config.oidc.enabled); - assert!(!config.oidc.auto_create_users); - - remove_var("CODEX_AUTH_OIDC_ENABLED"); - remove_var("CODEX_AUTH_OIDC_AUTO_CREATE_USERS"); - } - - #[test] - #[serial] - fn test_oidc_provider_role_mapping_env_override() { - remove_var("CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_ROLE_MAPPING_ADMIN"); - remove_var("CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_ROLE_MAPPING_MAINTAINER"); - remove_var("CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_ROLE_MAPPING_READER"); - - use crate::OidcProviderConfig; - - let mut provider = OidcProviderConfig { - display_name: "Authentik".to_string(), - issuer_url: "https://auth.example.com".to_string(), - client_id: "codex".to_string(), - client_secret: None, - client_secret_env: None, - scopes: vec![], - role_mapping: std::collections::HashMap::new(), - groups_claim: "groups".to_string(), - username_claim: "preferred_username".to_string(), - email_claim: "email".to_string(), - accepted_audiences: vec![], - }; - - set_var( - "CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_ROLE_MAPPING_ADMIN", - "codex-admins, administrators", - ); - set_var( - "CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_ROLE_MAPPING_MAINTAINER", - "codex-editors", - ); - set_var( - "CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_ROLE_MAPPING_READER", - "codex-users, users, guests", - ); - provider.apply_env_overrides("CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK"); - - assert_eq!( - provider.role_mapping.get("admin"), - Some(&vec![ - "codex-admins".to_string(), - "administrators".to_string() - ]) - ); - assert_eq!( - provider.role_mapping.get("maintainer"), - Some(&vec!["codex-editors".to_string()]) - ); - assert_eq!( - provider.role_mapping.get("reader"), - Some(&vec![ - "codex-users".to_string(), - "users".to_string(), - "guests".to_string() - ]) - ); - - remove_var("CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_ROLE_MAPPING_ADMIN"); - remove_var("CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_ROLE_MAPPING_MAINTAINER"); - remove_var("CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_ROLE_MAPPING_READER"); - } - - #[test] - #[serial] - fn test_oidc_provider_role_mapping_env_override_merges_with_existing() { - remove_var("CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_ROLE_MAPPING_ADMIN"); - - use crate::OidcProviderConfig; - - let mut role_mapping = std::collections::HashMap::new(); - role_mapping.insert("reader".to_string(), vec!["yaml-readers".to_string()]); - - let mut provider = OidcProviderConfig { - display_name: "Authentik".to_string(), - issuer_url: "https://auth.example.com".to_string(), - client_id: "codex".to_string(), - client_secret: None, - client_secret_env: None, - scopes: vec![], - role_mapping, - groups_claim: "groups".to_string(), - username_claim: "preferred_username".to_string(), - email_claim: "email".to_string(), - accepted_audiences: vec![], - }; - - // Add admin via env, reader stays from YAML - set_var( - "CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_ROLE_MAPPING_ADMIN", - "codex-admins", - ); - provider.apply_env_overrides("CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK"); - - assert_eq!( - provider.role_mapping.get("admin"), - Some(&vec!["codex-admins".to_string()]) - ); - // Existing reader mapping should still be present - assert_eq!( - provider.role_mapping.get("reader"), - Some(&vec!["yaml-readers".to_string()]) - ); - - remove_var("CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_ROLE_MAPPING_ADMIN"); - } - - // FilesConfig Environment Override Tests - - #[test] - #[serial] - fn test_files_config_env_override_all_fields() { - remove_var("CODEX_FILES_THUMBNAIL_DIR"); - remove_var("CODEX_FILES_UPLOADS_DIR"); - remove_var("CODEX_FILES_PLUGINS_DIR"); - - let mut config = FilesConfig { - thumbnail_dir: "data/thumbnails".to_string(), - uploads_dir: "data/uploads".to_string(), - plugins_dir: "data/plugins".to_string(), - }; - - set_var("CODEX_FILES_THUMBNAIL_DIR", "/custom/thumbs"); - set_var("CODEX_FILES_UPLOADS_DIR", "/custom/uploads"); - set_var("CODEX_FILES_PLUGINS_DIR", "/custom/plugins"); - config.apply_env_overrides("CODEX_FILES"); - - assert_eq!(config.thumbnail_dir, "/custom/thumbs"); - assert_eq!(config.uploads_dir, "/custom/uploads"); - assert_eq!(config.plugins_dir, "/custom/plugins"); - - remove_var("CODEX_FILES_THUMBNAIL_DIR"); - remove_var("CODEX_FILES_UPLOADS_DIR"); - remove_var("CODEX_FILES_PLUGINS_DIR"); - } - - #[test] - #[serial] - fn test_files_config_env_override_partial() { - remove_var("CODEX_FILES_THUMBNAIL_DIR"); - remove_var("CODEX_FILES_UPLOADS_DIR"); - remove_var("CODEX_FILES_PLUGINS_DIR"); - - let mut config = FilesConfig { - thumbnail_dir: "data/thumbnails".to_string(), - uploads_dir: "data/uploads".to_string(), - plugins_dir: "data/plugins".to_string(), - }; - - // Only override plugins_dir - set_var("CODEX_FILES_PLUGINS_DIR", "/mnt/storage/plugins"); - config.apply_env_overrides("CODEX_FILES"); - - assert_eq!(config.thumbnail_dir, "data/thumbnails"); // Unchanged - assert_eq!(config.uploads_dir, "data/uploads"); // Unchanged - assert_eq!(config.plugins_dir, "/mnt/storage/plugins"); - - remove_var("CODEX_FILES_PLUGINS_DIR"); - } - - #[test] - #[serial] - fn test_config_data_dir_env_override() { - remove_var("CODEX_DATA_DIR"); - - let mut config = Config::default(); - assert_eq!(config.data_dir, "data"); // Default - - set_var("CODEX_DATA_DIR", "/mnt/codex-data"); - config.apply_env_overrides("CODEX"); - - assert_eq!(config.data_dir, "/mnt/codex-data"); - - remove_var("CODEX_DATA_DIR"); - } - - #[test] - #[serial] - fn test_observability_env_override_all_fields() { - // Cover every leaf field at least once so a regression in the - // env_override impl is caught here rather than at runtime. - let vars = [ - ("CODEX_OBSERVABILITY_ENABLED", "true"), - ("CODEX_OBSERVABILITY_SERVICE_NAME", "codex-staging"), - ( - "CODEX_OBSERVABILITY_OTLP_ENDPOINT", - "https://otel.example.com:4317", - ), - ("CODEX_OBSERVABILITY_OTLP_PROTOCOL", "http/protobuf"), - ("CODEX_OBSERVABILITY_OTLP_TIMEOUT_MS", "9000"), - ( - "CODEX_OBSERVABILITY_OTLP_HEADERS", - "x-tenant=acme,x-key=secret", - ), - ( - "CODEX_OBSERVABILITY_OTLP_PROXY_ENDPOINT", - "http://collector.local:4318", - ), - ("CODEX_OBSERVABILITY_TRACES_ENABLED", "false"), - ("CODEX_OBSERVABILITY_TRACES_SAMPLE_RATIO", "0.3"), - ("CODEX_OBSERVABILITY_METRICS_ENABLED", "false"), - ("CODEX_OBSERVABILITY_METRICS_EXPORT_INTERVAL_MS", "60000"), - ("CODEX_OBSERVABILITY_BROWSER_ENABLED", "true"), - ("CODEX_OBSERVABILITY_BROWSER_PROXY_PATH", "/proxy"), - ("CODEX_OBSERVABILITY_BROWSER_SAMPLE_RATIO", "0.7"), - ]; - for (k, _) in vars.iter() { - remove_var(k); - } - for (k, v) in vars.iter() { - set_var(k, v); - } - - let mut config = crate::ObservabilityConfig::default(); - config.apply_env_overrides("CODEX_OBSERVABILITY"); - - assert!(config.enabled); - assert_eq!(config.service_name, "codex-staging"); - assert_eq!(config.otlp.endpoint, "https://otel.example.com:4317"); - assert!(matches!( - config.otlp.protocol, - crate::OtlpProtocol::HttpProtobuf - )); - assert_eq!(config.otlp.timeout_ms, 9000); - assert_eq!(config.otlp.headers.get("x-tenant"), Some(&"acme".into())); - assert_eq!(config.otlp.headers.get("x-key"), Some(&"secret".into())); - assert_eq!( - config.otlp.proxy_endpoint.as_deref(), - Some("http://collector.local:4318") - ); - assert!(!config.traces.enabled); - assert!((config.traces.sample_ratio - 0.3).abs() < f64::EPSILON); - assert!(!config.metrics.enabled); - assert_eq!(config.metrics.export_interval_ms, 60000); - assert!(config.browser.enabled); - assert_eq!(config.browser.proxy_path, "/proxy"); - assert!((config.browser.sample_ratio - 0.7).abs() < f64::EPSILON); - - for (k, _) in vars.iter() { - remove_var(k); - } - } -} diff --git a/crates/codex-config/src/lib.rs b/crates/codex-config/src/lib.rs index 0bc0865e3..03e80c3fa 100644 --- a/crates/codex-config/src/lib.rs +++ b/crates/codex-config/src/lib.rs @@ -4,7 +4,6 @@ //! the workspace-split plan. Has no dependencies on other Codex crates. mod env_audit; -mod env_override; mod keys; mod loader; mod redact; @@ -20,9 +19,9 @@ pub use types::{ }; pub use env_audit::{ - ENV_PREFIX, Finding, NON_CONFIG_VARS, audit, audit_env, audit_env_with_config, classify, + Finding, NON_CONFIG_VARS, audit, audit_env, audit_env_with_config, classify, secret_env_targets, v1_name_for, v2_name_for, }; -pub use env_override::EnvOverride; pub use keys::{KeyRegistry, registry}; +pub use loader::{ENV_NESTING_SEPARATOR, ENV_PREFIX}; pub use redact::{REDACTED, UNSET, redacted_value, redacted_yaml}; diff --git a/crates/codex-config/src/loader.rs b/crates/codex-config/src/loader.rs index ff1c1debf..928cb089c 100644 --- a/crates/codex-config/src/loader.rs +++ b/crates/codex-config/src/loader.rs @@ -1,15 +1,72 @@ +//! Layered configuration loading: struct defaults -> config file -> local +//! overlay -> environment. +//! +//! The config file may be YAML or TOML; the provider is chosen from the file +//! extension. A sibling `.local.` file (for example +//! `codex.yaml` -> `codex.local.yaml`) is merged on top of the base file when +//! present, so an operator can pin secrets and per-host overrides without +//! editing the committed config. +//! +//! Environment variables use the `CODEX_` prefix with `__` between nesting +//! levels: `CODEX_RATE_LIMIT__ANONYMOUS_RPS` sets `rate_limit.anonymous_rps`. +//! A single `_` separates words *within* one key, which is why the separator +//! has to be doubled: Codex v1 used a single `_` for both jobs, making +//! `CODEX_RATE_LIMIT_ANONYMOUS_RPS` impossible to split correctly without a +//! hand-maintained table of section names. + use super::types::Config; -use anyhow::Result; +use anyhow::{Context, Result}; +use figment::providers::{Env, Format, Serialized, Toml, Yaml}; +use figment::{Figment, Profile}; use std::fs; -use std::path::Path; +use std::path::{Path, PathBuf}; + +/// Prefix on every Codex environment variable. +pub const ENV_PREFIX: &str = "CODEX_"; + +/// Separator between nesting levels in an environment variable name. +pub const ENV_NESTING_SEPARATOR: &str = "__"; impl Config { - pub fn from_file>(path: P) -> Result { - let contents = fs::read_to_string(path)?; - let config: Config = serde_yaml::from_str(&contents)?; + /// Resolve configuration from `path`, its local overlay, and the + /// environment. + /// + /// A missing config file is not an error: defaults plus the environment + /// are a complete configuration on their own, which is what a container + /// with no mounted file relies on. + pub fn load>(path: P) -> Result { + let path = path.as_ref(); + + // The same layers twice: once over the defaults to produce the config, + // and once on their own to answer "did anyone actually set this key?". + // Without the second chain a value equal to its default is + // indistinguishable from an unset one, which is what path rooting + // needs to know. + let overrides = layers(path, Figment::new()); + let figment = layers(path, Figment::from(Serialized::defaults(Config::default()))); + + let mut config: Config = figment + .extract() + .with_context(|| format!("failed to parse configuration from {}", path.display()))?; + + config.root_paths_at_data_dir(&|key| overrides.find_value(key).is_ok()); + Ok(config) } + /// Read a single config file, ignoring the overlay and the environment. + /// + /// Only for callers that want the file's literal contents, such as reading + /// the other side's config during `codex copy`. Everything that configures + /// a running process wants [`Config::load`]. + pub fn from_file>(path: P) -> Result { + let path = path.as_ref(); + Figment::from(Serialized::defaults(Config::default())) + .merge(file_provider(path)) + .extract() + .with_context(|| format!("failed to parse configuration from {}", path.display())) + } + pub fn to_file>(&self, path: P) -> Result<()> { let yaml = serde_yaml::to_string(self)?; fs::write(path, yaml)?; @@ -17,168 +74,449 @@ impl Config { } } +/// Stack the file, overlay and environment layers onto `base`. +fn layers(path: &Path, base: Figment) -> Figment { + let mut figment = base; + + if path.exists() { + figment = figment.merge(file_provider(path)); + } + + if let Some(local) = local_overlay_path(path) + && local.exists() + { + figment = figment.merge(file_provider(&local)); + } + + figment.merge(Env::prefixed(ENV_PREFIX).split(ENV_NESTING_SEPARATOR)) +} + +/// Pick a provider by file extension. Anything that is not `.toml` is read as +/// YAML, which keeps `codex.yaml`, `codex.yml` and extensionless paths working. +fn file_provider(path: &Path) -> Figment { + match path.extension().and_then(|ext| ext.to_str()) { + Some(ext) if ext.eq_ignore_ascii_case("toml") => { + Figment::from(Toml::file(path).profile(Profile::Default)) + } + _ => Figment::from(Yaml::file(path).profile(Profile::Default)), + } +} + +/// `config/codex.yaml` -> `config/codex.local.yaml`. +/// +/// Returns `None` for a path with no extension, where there is no sensible +/// place to put the `.local` infix. +fn local_overlay_path(path: &Path) -> Option { + let extension = path.extension()?.to_str()?; + let stem = path.file_stem()?.to_str()?; + let mut local = path.to_path_buf(); + local.set_file_name(format!("{stem}.local.{extension}")); + Some(local) +} + #[cfg(test)] +// `figment::Error` is a large type and every `Jail` closure returns it, which +// is the shape figment's test harness requires. +#[allow(clippy::result_large_err)] mod tests { use super::*; - use crate::{ - ApiConfig, ApplicationConfig, AuthConfig, DatabaseConfig, DatabaseType, EmailConfig, - FilesConfig, KomgaApiConfig, KoreaderApiConfig, LoggingConfig, ObservabilityConfig, - PdfConfig, PdfHandleCacheConfig, RateLimitConfig, SQLiteConfig, ScannerConfig, - SchedulerConfig, TaskConfig, - }; - use tempfile::NamedTempFile; + use crate::DatabaseType; + use crate::types::LogLevel; + use figment::Jail; #[test] - fn test_config_from_file() { - let yaml_content = r#" -database: - db_type: sqlite - sqlite: - path: ./test.db -application: - host: 127.0.0.1 - port: 3000 -"#; - - let temp_file = NamedTempFile::new().unwrap(); - std::fs::write(temp_file.path(), yaml_content).unwrap(); - - let config = Config::from_file(temp_file.path()).unwrap(); - - // Application name moved to database settings - assert_eq!(config.application.host, "127.0.0.1"); - assert_eq!(config.application.port, 3000); - assert!(matches!(config.database.db_type, DatabaseType::SQLite)); - } - - #[test] - fn test_config_to_file() { - let config = Config { - data_dir: "data".to_string(), - database: DatabaseConfig { - db_type: DatabaseType::SQLite, - postgres: None, - sqlite: Some(SQLiteConfig { - path: "./codex.db".to_string(), - pragmas: None, - ..SQLiteConfig::default() - }), - }, - application: ApplicationConfig { - host: "0.0.0.0".to_string(), - port: 8080, - ..Default::default() - }, - logging: LoggingConfig::default(), - auth: AuthConfig::default(), - api: ApiConfig::default(), - email: EmailConfig::default(), - task: TaskConfig::default(), - scanner: ScannerConfig::default(), - scheduler: SchedulerConfig::default(), - files: FilesConfig::default(), - pdf: PdfConfig::default(), - pdf_handle_cache: PdfHandleCacheConfig::default(), - komga_api: KomgaApiConfig::default(), - koreader_api: KoreaderApiConfig::default(), - rate_limit: RateLimitConfig::default(), - observability: ObservabilityConfig::default(), - }; - - let temp_file = NamedTempFile::new().unwrap(); - config.to_file(temp_file.path()).unwrap(); - - let loaded_config = Config::from_file(temp_file.path()).unwrap(); - - // Application name moved to database settings - assert_eq!(loaded_config.application.port, 8080); - assert!(matches!( - loaded_config.database.db_type, - DatabaseType::SQLite - )); - } - - #[test] - fn test_config_from_invalid_file() { - let result = Config::from_file("/nonexistent/path/to/file.yaml"); - assert!(result.is_err()); - } - - #[test] - fn test_config_from_malformed_yaml() { - let yaml_content = "this is not valid yaml: {{{}"; - - let temp_file = NamedTempFile::new().unwrap(); - std::fs::write(temp_file.path(), yaml_content).unwrap(); - - let result = Config::from_file(temp_file.path()); - assert!(result.is_err()); - } - - #[test] - fn test_config_with_task_and_scanner_sections() { - let yaml_content = r#" + fn defaults_apply_when_no_file_exists() { + Jail::expect_with(|jail| { + let config = Config::load(jail.directory().join("absent.yaml")).unwrap(); + assert_eq!(config.application.host, "0.0.0.0"); + assert_eq!(config.application.port, 8080); + assert_eq!(config.task.worker_count, 2); + Ok(()) + }); + } + + #[test] + fn the_config_file_overrides_defaults() { + Jail::expect_with(|jail| { + jail.create_file("codex.yaml", "application:\n port: 9000\n")?; + let config = Config::load(jail.directory().join("codex.yaml")).unwrap(); + assert_eq!(config.application.port, 9000); + assert_eq!( + config.application.host, "0.0.0.0", + "untouched key keeps its default" + ); + Ok(()) + }); + } + + #[test] + fn a_toml_file_is_read_as_toml() { + Jail::expect_with(|jail| { + jail.create_file("codex.toml", "[application]\nport = 9100\n")?; + let config = Config::load(jail.directory().join("codex.toml")).unwrap(); + assert_eq!(config.application.port, 9100); + Ok(()) + }); + } + + #[test] + fn the_local_overlay_beats_the_base_file() { + Jail::expect_with(|jail| { + jail.create_file( + "codex.yaml", + "application:\n port: 9000\n host: 1.1.1.1\n", + )?; + jail.create_file("codex.local.yaml", "application:\n port: 9999\n")?; + + let config = Config::load(jail.directory().join("codex.yaml")).unwrap(); + + assert_eq!(config.application.port, 9999); + assert_eq!( + config.application.host, "1.1.1.1", + "the overlay merges field by field rather than replacing the section" + ); + Ok(()) + }); + } + + #[test] + fn the_local_overlay_is_optional() { + Jail::expect_with(|jail| { + jail.create_file("codex.yaml", "application:\n port: 9001\n")?; + let config = Config::load(jail.directory().join("codex.yaml")).unwrap(); + assert_eq!(config.application.port, 9001); + Ok(()) + }); + } + + /// The whole precedence chain in one test, one layer per level. + #[test] + fn the_environment_wins_over_every_file_layer() { + Jail::expect_with(|jail| { + jail.create_file("codex.yaml", "application:\n port: 2\n")?; + jail.create_file("codex.local.yaml", "application:\n port: 3\n")?; + jail.set_env("CODEX_APPLICATION__PORT", "4"); + + let config = Config::load(jail.directory().join("codex.yaml")).unwrap(); + + assert_eq!(config.application.port, 4); + Ok(()) + }); + } + + #[test] + fn nested_sections_use_one_separator_per_level() { + Jail::expect_with(|jail| { + jail.set_env("CODEX_RATE_LIMIT__ANONYMOUS_RPS", "77"); + jail.set_env("CODEX_OBSERVABILITY__OTLP__TIMEOUT_MS", "1234"); + jail.set_env("CODEX_PDF_HANDLE_CACHE__CAPACITY", "999"); + + let config = Config::load(jail.directory().join("absent.yaml")).unwrap(); + + assert_eq!(config.rate_limit.anonymous_rps, 77); + assert_eq!(config.observability.otlp.timeout_ms, 1234); + assert_eq!(config.pdf_handle_cache.capacity, 999); + Ok(()) + }); + } + + /// v1 spelling must NOT be honoured. It is reported by the validator with + /// its replacement instead of quietly half-working. + #[test] + fn the_old_flat_names_are_not_read() { + Jail::expect_with(|jail| { + jail.set_env("CODEX_APPLICATION_PORT", "4321"); + let config = Config::load(jail.directory().join("absent.yaml")).unwrap(); + assert_eq!(config.application.port, 8080); + Ok(()) + }); + } + + /// The env layer can create a subtree the file never mentioned. In v1 this + /// silently did nothing, because the override was only applied when + /// `database.postgres` was already `Some`, and `display_database_config` + /// then unwrapped the `None`. + #[test] + fn the_environment_can_introduce_the_postgres_subtree() { + Jail::expect_with(|jail| { + jail.create_file("codex.yaml", "database:\n db_type: sqlite\n")?; + jail.set_env("CODEX_DATABASE__DB_TYPE", "postgres"); + jail.set_env("CODEX_DATABASE__POSTGRES__HOST", "db.internal"); + jail.set_env("CODEX_DATABASE__POSTGRES__PASSWORD", "hunter2"); + + let config = Config::load(jail.directory().join("codex.yaml")).unwrap(); + + assert_eq!(config.database.db_type, DatabaseType::Postgres); + let postgres = config + .database + .postgres + .expect("env should have created the postgres section"); + assert_eq!(postgres.host, "db.internal"); + assert_eq!(postgres.password, "hunter2"); + assert_eq!( + postgres.port, 5432, + "unset keys still fall back to defaults" + ); + Ok(()) + }); + } + + #[test] + fn maps_round_trip_through_the_file_layer() { + Jail::expect_with(|jail| { + jail.create_file( + "codex.yaml", + r#" database: - db_type: sqlite sqlite: - path: ./test.db -task: - worker_count: 6 -scanner: - max_concurrent_scans: 3 -"#; - - let temp_file = NamedTempFile::new().unwrap(); - std::fs::write(temp_file.path(), yaml_content).unwrap(); - - let config = Config::from_file(temp_file.path()).unwrap(); - - assert_eq!(config.task.worker_count, 6); - assert_eq!(config.scanner.max_concurrent_scans, 3); - } - - #[test] - fn test_config_serialization_includes_task_and_scanner() { - let config = Config { - data_dir: "data".to_string(), - database: DatabaseConfig { - db_type: DatabaseType::SQLite, - postgres: None, - sqlite: Some(SQLiteConfig { - path: "./codex.db".to_string(), - pragmas: None, - ..SQLiteConfig::default() - }), - }, - application: ApplicationConfig { - host: "127.0.0.1".to_string(), - port: 8080, - ..Default::default() - }, - logging: LoggingConfig::default(), - auth: AuthConfig::default(), - api: ApiConfig::default(), - email: EmailConfig::default(), - task: TaskConfig { worker_count: 8 }, - scanner: ScannerConfig { - max_concurrent_scans: 4, - }, - scheduler: SchedulerConfig::default(), - files: FilesConfig::default(), - pdf: PdfConfig::default(), - pdf_handle_cache: PdfHandleCacheConfig::default(), - komga_api: KomgaApiConfig::default(), - koreader_api: KoreaderApiConfig::default(), - rate_limit: RateLimitConfig::default(), - observability: ObservabilityConfig::default(), - }; - - let temp_file = NamedTempFile::new().unwrap(); - config.to_file(temp_file.path()).unwrap(); - - let loaded_config = Config::from_file(temp_file.path()).unwrap(); - - assert_eq!(loaded_config.task.worker_count, 8); - assert_eq!(loaded_config.scanner.max_concurrent_scans, 4); + pragmas: + journal_mode: DELETE +auth: + oidc: + providers: + authentik: + display_name: Authentik + issuer_url: https://idp.example.com + client_id: codex +"#, + )?; + + let config = Config::load(jail.directory().join("codex.yaml")).unwrap(); + + let pragmas = config.database.sqlite.unwrap().pragmas.unwrap(); + assert_eq!( + pragmas.get("journal_mode").map(String::as_str), + Some("DELETE") + ); + + let provider = &config.auth.oidc.providers["authentik"]; + assert_eq!(provider.issuer_url, "https://idp.example.com"); + assert_eq!( + provider.groups_claim, "groups", + "provider fields absent from the file keep their serde defaults" + ); + Ok(()) + }); + } + + #[test] + fn enums_parse_from_both_the_file_and_the_environment() { + Jail::expect_with(|jail| { + jail.create_file("codex.yaml", "logging:\n level: debug\n")?; + let config = Config::load(jail.directory().join("codex.yaml")).unwrap(); + assert!(matches!(config.logging.level, LogLevel::Debug)); + + jail.set_env("CODEX_LOGGING__LEVEL", "warn"); + let config = Config::load(jail.directory().join("codex.yaml")).unwrap(); + assert!(matches!(config.logging.level, LogLevel::Warn)); + Ok(()) + }); + } + + #[test] + fn a_malformed_file_is_an_error_naming_the_path() { + Jail::expect_with(|jail| { + jail.create_file("codex.yaml", "this is not valid yaml: {{{}")?; + let error = Config::load(jail.directory().join("codex.yaml")).unwrap_err(); + assert!( + error.to_string().contains("codex.yaml"), + "error should name the file: {error}" + ); + Ok(()) + }); + } + + // ---- path rooting under data_dir (replaces the old sentinel logic) ---- + + #[test] + fn absent_paths_are_rooted_at_data_dir() { + Jail::expect_with(|jail| { + jail.create_file("codex.yaml", "data_dir: /var/lib/codex\n")?; + + let config = Config::load(jail.directory().join("codex.yaml")).unwrap(); + + assert_eq!(config.files.thumbnail_dir, "/var/lib/codex/thumbnails"); + assert_eq!(config.files.uploads_dir, "/var/lib/codex/uploads"); + assert_eq!(config.files.plugins_dir, "/var/lib/codex/plugins"); + assert_eq!(config.pdf.cache_dir, "/var/lib/codex/cache"); + assert_eq!( + config.database.sqlite.unwrap().path, + "/var/lib/codex/codex.db" + ); + Ok(()) + }); + } + + #[test] + fn an_explicit_path_survives_a_different_data_dir() { + Jail::expect_with(|jail| { + jail.create_file( + "codex.yaml", + "data_dir: /var/lib/codex\nfiles:\n thumbnail_dir: /mnt/thumbs\n", + )?; + + let config = Config::load(jail.directory().join("codex.yaml")).unwrap(); + + assert_eq!(config.files.thumbnail_dir, "/mnt/thumbs"); + assert_eq!(config.files.uploads_dir, "/var/lib/codex/uploads"); + Ok(()) + }); + } + + /// The case the old sentinel got wrong: writing the literal default is an + /// explicit choice, and used to be silently rewritten. + #[test] + fn writing_the_literal_default_path_is_respected() { + Jail::expect_with(|jail| { + jail.create_file( + "codex.yaml", + "data_dir: /var/lib/codex\nfiles:\n thumbnail_dir: data/thumbnails\n", + )?; + + let config = Config::load(jail.directory().join("codex.yaml")).unwrap(); + + assert_eq!( + config.files.thumbnail_dir, "data/thumbnails", + "an explicitly written path must not be rewritten, even when it \ + happens to equal the old hardcoded default" + ); + Ok(()) + }); + } + + #[test] + fn the_environment_can_set_a_path_directly() { + Jail::expect_with(|jail| { + jail.create_file("codex.yaml", "data_dir: /var/lib/codex\n")?; + jail.set_env("CODEX_FILES__THUMBNAIL_DIR", "/mnt/env-thumbs"); + + let config = Config::load(jail.directory().join("codex.yaml")).unwrap(); + + assert_eq!(config.files.thumbnail_dir, "/mnt/env-thumbs"); + Ok(()) + }); + } + + #[test] + fn the_default_data_dir_produces_the_historical_layout() { + Jail::expect_with(|jail| { + let config = Config::load(jail.directory().join("absent.yaml")).unwrap(); + assert_eq!(config.files.thumbnail_dir, "data/thumbnails"); + assert_eq!(config.pdf.cache_dir, "data/cache"); + assert_eq!(config.database.sqlite.unwrap().path, "data/codex.db"); + Ok(()) + }); + } + + /// Every scalar setting must be reachable from the environment. + /// + /// This replaces the per-key tests that lived in the deleted + /// `env_override.rs`. Rather than a hand-written list that goes stale, it + /// walks the key registry, sets each key through its `__` name with a + /// value of the right shape, and checks the loaded config actually + /// changed at that path. + #[test] + fn every_scalar_setting_is_reachable_from_the_environment() { + use serde_json::Value; + + /// Enum-valued keys need a valid variant, not an arbitrary string. + /// The third column is the canonical spelling the value serializes + /// back to, which differs from the input wherever an alias is used. + const ENUM_VALUES: &[(&str, &str, &str)] = &[ + ("database.db_type", "postgresql", "postgres"), + ("logging.level", "warn", "warn"), + ("observability.otlp.protocol", "http/json", "http-json"), + ("auth.oidc.default_role", "admin", "admin"), + ]; + + fn at<'v>(value: &'v Value, path: &str) -> Option<&'v Value> { + path.split('.').try_fold(value, |node, seg| node.get(seg)) + } + + let defaults = serde_json::to_value(Config::default()).unwrap(); + let mut checked = 0usize; + + for key in crate::registry().exact() { + // Only keys present in the default tree can be shape-inferred. + // Subtrees that default to absent (postgres) are covered by + // `the_environment_can_introduce_the_postgres_subtree`. + let Some(current) = at(&defaults, key) else { + continue; + }; + + let (raw, expected): (String, Value) = match ENUM_VALUES + .iter() + .find(|(k, _, _)| k == key) + { + Some((_, input, canonical)) => ((*input).to_string(), Value::from(*canonical)), + None => match current { + Value::Bool(b) => ((!b).to_string(), Value::Bool(!b)), + Value::Number(n) if n.is_f64() => ("0.25".to_string(), Value::from(0.25f64)), + Value::Number(n) => { + let next = n.as_u64().unwrap_or(0) + 7; + (next.to_string(), Value::from(next)) + } + Value::String(_) | Value::Null => { + ("env-probe".to_string(), Value::from("env-probe")) + } + // Arrays and maps need the lenient env forms, which are + // covered by their own tests. + _ => continue, + }, + }; + + let var = crate::v2_name_for(key); + Jail::expect_with(|jail| { + jail.set_env(&var, &raw); + let config = Config::load(jail.directory().join("absent.yaml")) + .unwrap_or_else(|e| panic!("{var}={raw} should load: {e:#}")); + let loaded = serde_json::to_value(config).unwrap(); + assert_eq!( + at(&loaded, key), + Some(&expected), + "setting {var} did not reach `{key}`" + ); + Ok(()) + }); + checked += 1; + } + + assert!( + checked > 60, + "expected broad coverage of the settable surface, only checked {checked}" + ); + } + + #[test] + fn overlay_path_gets_a_local_infix() { + assert_eq!( + local_overlay_path(Path::new("config/codex.yaml")), + Some(PathBuf::from("config/codex.local.yaml")) + ); + assert_eq!( + local_overlay_path(Path::new("/etc/codex/config.docker.toml")), + Some(PathBuf::from("/etc/codex/config.docker.local.toml")) + ); + assert_eq!(local_overlay_path(Path::new("codex")), None); + } + + /// Every shipped example must still load. These are what operators copy. + #[test] + fn the_bundled_example_configs_load() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .parent() + .unwrap(); + for name in [ + "config.docker.yaml", + "config.kubernetes.yaml", + "config.sqlite.yaml", + "config.screenshots.yaml", + ] { + let path = root.join("config").join(name); + assert!(path.exists(), "missing example config {}", path.display()); + Config::from_file(&path).unwrap_or_else(|e| panic!("{name} should load: {e:#}")); + } } } diff --git a/crates/codex-config/src/testdata/default_config.json b/crates/codex-config/src/testdata/default_config.json new file mode 100644 index 000000000..6a5388342 --- /dev/null +++ b/crates/codex-config/src/testdata/default_config.json @@ -0,0 +1,138 @@ +{ + "data_dir": "data", + "database": { + "db_type": "sqlite", + "sqlite": { + "path": "data/codex.db", + "pragmas": { + "journal_mode": "WAL", + "foreign_keys": "ON" + }, + "max_connections": 64, + "min_connections": 2, + "batch_fan_out": 4, + "background_max_connections": 4, + "acquire_timeout_seconds": 30, + "idle_timeout_seconds": 300, + "max_lifetime_seconds": 1800, + "operation_deadline_seconds": 30 + } + }, + "application": { + "host": "0.0.0.0", + "port": 8080 + }, + "logging": { + "level": "info", + "console": true, + "file": null + }, + "auth": { + "jwt_secret": "INSECURE_DEFAULT_SECRET_CHANGE_IN_PRODUCTION", + "jwt_expiry_hours": 24, + "refresh_token_enabled": true, + "refresh_token_expiry_days": 30, + "email_confirmation_required": false, + "argon2_memory_cost": 19456, + "argon2_time_cost": 2, + "argon2_parallelism": 1, + "oidc": { + "enabled": false, + "auto_create_users": true, + "default_role": "reader", + "allowed_redirect_uris": [], + "providers": {} + } + }, + "api": { + "base_path": "/api/v1", + "enable_api_docs": false, + "api_docs_path": "/docs", + "cors_enabled": true, + "cors_origins": [ + "*" + ], + "max_page_size": 100 + }, + "email": { + "smtp_host": "localhost", + "smtp_port": 587, + "smtp_username": "", + "smtp_password": "", + "smtp_from_email": "noreply@example.com", + "smtp_from_name": "Codex", + "verification_token_expiry_hours": 24 + }, + "task": { + "worker_count": 2 + }, + "scanner": { + "max_concurrent_scans": 2 + }, + "scheduler": { + "timezone": "UTC" + }, + "files": { + "thumbnail_dir": "data/thumbnails", + "uploads_dir": "data/uploads", + "plugins_dir": "data/plugins" + }, + "pdf": { + "pdfium_library_path": null, + "render_dpi": 150, + "jpeg_quality": 85, + "cache_rendered_pages": true, + "cache_dir": "data/cache" + }, + "pdf_handle_cache": { + "enabled": true, + "capacity": 256, + "idle_ttl_minutes": 15, + "sweep_interval_seconds": 60 + }, + "komga_api": { + "enabled": false, + "prefix": "komga" + }, + "koreader_api": { + "enabled": false + }, + "rate_limit": { + "enabled": true, + "anonymous_rps": 10, + "anonymous_burst": 50, + "authenticated_rps": 50, + "authenticated_burst": 200, + "exempt_paths": [ + "/health", + "/api/v1/events", + "/api/v1/events/**" + ], + "cleanup_interval_secs": 60, + "bucket_ttl_secs": 300 + }, + "observability": { + "enabled": false, + "service_name": "codex", + "otlp": { + "endpoint": "", + "protocol": "grpc", + "headers": {}, + "timeout_ms": 5000, + "proxy_endpoint": null + }, + "traces": { + "enabled": true, + "sample_ratio": 1.0 + }, + "metrics": { + "enabled": true, + "export_interval_ms": 30000 + }, + "browser": { + "enabled": false, + "proxy_path": "/api/v1/observability/otlp", + "sample_ratio": 0.1 + } + } +} diff --git a/crates/codex-config/src/types.rs b/crates/codex-config/src/types.rs index aa64a0302..487923ad1 100644 --- a/crates/codex-config/src/types.rs +++ b/crates/codex-config/src/types.rs @@ -1,4 +1,3 @@ -use super::env_override::{env_bool_or, env_or, env_string_opt, parse_csv_list}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; @@ -32,7 +31,7 @@ fn default_komga_prefix() -> String { /// Configuration for the KOReader sync API /// Enables KOReader e-readers to sync reading progress with Codex -#[derive(Debug, Serialize, Deserialize, Clone)] +#[derive(Debug, Serialize, Deserialize, Clone, Default)] #[serde(default)] pub struct KoreaderApiConfig { /// Enable KOReader sync API endpoints @@ -40,14 +39,6 @@ pub struct KoreaderApiConfig { pub enabled: bool, } -impl Default for KoreaderApiConfig { - fn default() -> Self { - Self { - enabled: env_bool_or("CODEX_KOREADER_API_ENABLED", false), - } - } -} - /// Configuration for API rate limiting /// Uses token bucket algorithm with per-client tracking #[derive(Debug, Serialize, Deserialize, Clone)] @@ -209,13 +200,11 @@ pub struct OidcConfig { impl Default for OidcConfig { fn default() -> Self { Self { - enabled: env_bool_or("CODEX_AUTH_OIDC_ENABLED", false), - auto_create_users: env_bool_or("CODEX_AUTH_OIDC_AUTO_CREATE_USERS", true), + enabled: false, + auto_create_users: true, default_role: OidcDefaultRole::Reader, - redirect_uri_base: std::env::var("CODEX_AUTH_OIDC_REDIRECT_URI_BASE").ok(), - allowed_redirect_uris: env_string_opt("CODEX_AUTH_OIDC_ALLOWED_REDIRECT_URIS") - .map(|s| parse_csv_list(&s)) - .unwrap_or_default(), + redirect_uri_base: None, + allowed_redirect_uris: Vec::new(), providers: HashMap::new(), } } @@ -224,16 +213,14 @@ impl Default for OidcConfig { impl Default for RateLimitConfig { fn default() -> Self { Self { - enabled: env_bool_or("CODEX_RATE_LIMIT_ENABLED", true), - anonymous_rps: env_or("CODEX_RATE_LIMIT_ANONYMOUS_RPS", 10), - anonymous_burst: env_or("CODEX_RATE_LIMIT_ANONYMOUS_BURST", 50), - authenticated_rps: env_or("CODEX_RATE_LIMIT_AUTHENTICATED_RPS", 50), - authenticated_burst: env_or("CODEX_RATE_LIMIT_AUTHENTICATED_BURST", 200), - exempt_paths: env_string_opt("CODEX_RATE_LIMIT_EXEMPT_PATHS") - .map(|s| s.split(',').map(|p| p.trim().to_string()).collect()) - .unwrap_or_else(default_exempt_paths), - cleanup_interval_secs: env_or("CODEX_RATE_LIMIT_CLEANUP_INTERVAL_SECS", 60), - bucket_ttl_secs: env_or("CODEX_RATE_LIMIT_BUCKET_TTL_SECS", 300), + enabled: true, + anonymous_rps: 10, + anonymous_burst: 50, + authenticated_rps: 50, + authenticated_burst: 200, + exempt_paths: default_exempt_paths(), + cleanup_interval_secs: 60, + bucket_ttl_secs: 300, } } } @@ -241,17 +228,15 @@ impl Default for RateLimitConfig { impl Default for KomgaApiConfig { fn default() -> Self { Self { - enabled: env_bool_or("CODEX_KOMGA_API_ENABLED", false), - prefix: env_string_opt("CODEX_KOMGA_API_PREFIX").unwrap_or_else(default_komga_prefix), + enabled: false, + prefix: default_komga_prefix(), } } } impl Default for TaskConfig { fn default() -> Self { - Self { - worker_count: env_or("CODEX_TASK_WORKER_COUNT", 2), - } + Self { worker_count: 2 } } } @@ -298,7 +283,7 @@ pub struct Config { } fn default_data_dir() -> String { - env_string_opt("CODEX_DATA_DIR").unwrap_or_else(|| "data".to_string()) + "data".to_string() } /// Default sub-directory names under data_dir @@ -309,100 +294,75 @@ const DEFAULT_CACHE_SUBDIR: &str = "cache"; const DEFAULT_SQLITE_FILENAME: &str = "codex.db"; impl Config { - /// Resolve sub-directory paths relative to `data_dir`. + /// Root the per-kind data directories under `data_dir`. /// - /// For each sub-path (thumbnail_dir, uploads_dir, plugins_dir, cache_dir, sqlite path), - /// if the value matches the old hardcoded default (e.g., "data/thumbnails") AND no - /// explicit env var override is set for that field, replace it with `{data_dir}/{subdir}`. + /// Every sub-path (`files.*_dir`, `pdf.cache_dir`, the SQLite file) defaults + /// to a subdirectory of `data_dir`, so setting `data_dir: /var/lib/codex` + /// moves all of them at once. A path the operator set explicitly is left + /// exactly as written. /// - /// This ensures backward compatibility: users who never set `data_dir` get the same - /// paths as before ("data/thumbnails"), while users who set `data_dir: /var/lib/codex` - /// get "/var/lib/codex/thumbnails" automatically. + /// `was_set` answers "did the config file, the local overlay, or the + /// environment provide this key?" for a dotted path. It has to come from + /// the loader, because by the time the config is deserialized an explicit + /// value and a default are indistinguishable. /// - /// Explicit overrides (env vars or non-default config values) always take precedence. - pub fn resolve_data_dir(&mut self) { - let data_dir = &self.data_dir; - - // Helper: build the derived path from data_dir - let derive = |subdir: &str| -> String { format!("{}/{}", data_dir, subdir) }; - - // Helper: check if a field uses the old hardcoded default ("data/{subdir}") - let is_old_default = - |value: &str, subdir: &str| -> bool { value == format!("data/{}", subdir) }; - - // Resolve files.thumbnail_dir - if is_old_default(&self.files.thumbnail_dir, DEFAULT_THUMBNAILS_SUBDIR) - && env_string_opt("CODEX_FILES_THUMBNAIL_DIR").is_none() - { - self.files.thumbnail_dir = derive(DEFAULT_THUMBNAILS_SUBDIR); - } - - // Resolve files.uploads_dir - if is_old_default(&self.files.uploads_dir, DEFAULT_UPLOADS_SUBDIR) - && env_string_opt("CODEX_FILES_UPLOADS_DIR").is_none() - { - self.files.uploads_dir = derive(DEFAULT_UPLOADS_SUBDIR); - } - - // Resolve files.plugins_dir - if is_old_default(&self.files.plugins_dir, DEFAULT_PLUGINS_SUBDIR) - && env_string_opt("CODEX_FILES_PLUGINS_DIR").is_none() - { - self.files.plugins_dir = derive(DEFAULT_PLUGINS_SUBDIR); - } + /// The previous implementation guessed instead: it compared each field + /// against the literal string `data/` and treated a match as + /// "unset". That silently rewrote the path of anyone who wrote + /// `thumbnail_dir: data/thumbnails` on purpose. + pub(crate) fn root_paths_at_data_dir(&mut self, was_set: &dyn Fn(&str) -> bool) { + let data_dir = self.data_dir.clone(); + let derive_unless_set = |key: &str, subdir: &str, slot: &mut String| { + if !was_set(key) { + *slot = format!("{data_dir}/{subdir}"); + } + }; - // Resolve pdf.cache_dir - if is_old_default(&self.pdf.cache_dir, DEFAULT_CACHE_SUBDIR) - && env_string_opt("CODEX_PDF_CACHE_DIR").is_none() - { - self.pdf.cache_dir = derive(DEFAULT_CACHE_SUBDIR); - } + derive_unless_set( + "files.thumbnail_dir", + DEFAULT_THUMBNAILS_SUBDIR, + &mut self.files.thumbnail_dir, + ); + derive_unless_set( + "files.uploads_dir", + DEFAULT_UPLOADS_SUBDIR, + &mut self.files.uploads_dir, + ); + derive_unless_set( + "files.plugins_dir", + DEFAULT_PLUGINS_SUBDIR, + &mut self.files.plugins_dir, + ); + derive_unless_set( + "pdf.cache_dir", + DEFAULT_CACHE_SUBDIR, + &mut self.pdf.cache_dir, + ); - // Resolve database.sqlite.path - if let Some(ref mut sqlite_config) = self.database.sqlite { - let old_default = format!("data/{}", DEFAULT_SQLITE_FILENAME); - if sqlite_config.path == old_default - && env_string_opt("CODEX_DATABASE_SQLITE_PATH").is_none() - { - sqlite_config.path = derive(DEFAULT_SQLITE_FILENAME); - } + if let Some(sqlite) = self.database.sqlite.as_mut() { + derive_unless_set( + "database.sqlite.path", + DEFAULT_SQLITE_FILENAME, + &mut sqlite.path, + ); } } } impl Default for Config { fn default() -> Self { - use std::env; - let mut pragmas = HashMap::new(); pragmas.insert("foreign_keys".to_string(), "ON".to_string()); pragmas.insert("journal_mode".to_string(), "WAL".to_string()); - // Determine database type from environment or use SQLite as default - let db_type = env::var("CODEX_DATABASE_DB_TYPE") - .ok() - .and_then(|t| { - if t.eq_ignore_ascii_case("postgres") || t.eq_ignore_ascii_case("postgresql") { - Some(DatabaseType::Postgres) - } else if t.eq_ignore_ascii_case("sqlite") { - Some(DatabaseType::SQLite) - } else { - None - } - }) - .unwrap_or(DatabaseType::SQLite); - - // Build database config based on type - let (postgres_config, sqlite_config) = match db_type { - DatabaseType::Postgres => (Some(PostgresConfig::default()), None), - DatabaseType::SQLite => ( - None, - Some(SQLiteConfig { - pragmas: Some(pragmas), - ..SQLiteConfig::default() - }), - ), - }; + let db_type = DatabaseType::SQLite; + let (postgres_config, sqlite_config) = ( + None, + Some(SQLiteConfig { + pragmas: Some(pragmas), + ..SQLiteConfig::default() + }), + ); Self { data_dir: default_data_dir(), @@ -453,10 +413,7 @@ impl Default for AuthConfig { jwt_expiry_hours: 24, refresh_token_enabled: true, refresh_token_expiry_days: 30, - email_confirmation_required: env_bool_or( - "CODEX_AUTH_EMAIL_CONFIRMATION_REQUIRED", - false, - ), + email_confirmation_required: false, argon2_memory_cost: 19456, argon2_time_cost: 2, argon2_parallelism: 1, @@ -479,16 +436,12 @@ pub struct ApiConfig { impl Default for ApiConfig { fn default() -> Self { Self { - base_path: env_string_opt("CODEX_API_BASE_PATH") - .unwrap_or_else(|| "/api/v1".to_string()), - enable_api_docs: env_bool_or("CODEX_API_ENABLE_API_DOCS", false), - api_docs_path: env_string_opt("CODEX_API_DOCS_PATH") - .unwrap_or_else(|| "/docs".to_string()), - cors_enabled: env_bool_or("CODEX_API_CORS_ENABLED", true), - cors_origins: env_string_opt("CODEX_API_CORS_ORIGINS") - .map(|s| s.split(',').map(|s| s.trim().to_string()).collect()) - .unwrap_or_else(|| vec!["*".to_string()]), - max_page_size: env_or("CODEX_API_MAX_PAGE_SIZE", 100), + base_path: "/api/v1".to_string(), + enable_api_docs: false, + api_docs_path: "/docs".to_string(), + cors_enabled: true, + cors_origins: vec!["*".to_string()], + max_page_size: 100, } } } @@ -573,27 +526,10 @@ impl DatabaseConfig { impl Default for DatabaseConfig { fn default() -> Self { - use std::env; - - // Determine database type from environment or use SQLite as default - let db_type = env::var("CODEX_DATABASE_DB_TYPE") - .ok() - .and_then(|t| { - if t.eq_ignore_ascii_case("postgres") || t.eq_ignore_ascii_case("postgresql") { - Some(DatabaseType::Postgres) - } else if t.eq_ignore_ascii_case("sqlite") { - Some(DatabaseType::SQLite) - } else { - None - } - }) - .unwrap_or(DatabaseType::SQLite); - - // Build database config based on type - let (postgres_config, sqlite_config) = match db_type { - DatabaseType::Postgres => (Some(PostgresConfig::default()), None), - DatabaseType::SQLite => (None, Some(SQLiteConfig::default())), - }; + // SQLite is the default engine; a `postgres` config selects the other + // branch through deserialization, not through the environment. + let db_type = DatabaseType::SQLite; + let (postgres_config, sqlite_config) = (None, Some(SQLiteConfig::default())); Self { db_type, @@ -607,6 +543,9 @@ impl Default for DatabaseConfig { #[serde(rename_all = "lowercase")] #[derive(Default)] pub enum DatabaseType { + /// `postgresql` is accepted too: the v1 environment override matched both + /// spellings, and connection strings commonly use the longer one. + #[serde(alias = "postgresql")] Postgres, #[default] SQLite, @@ -677,27 +616,20 @@ pub struct PostgresConfig { impl Default for PostgresConfig { fn default() -> Self { Self { - host: env_string_opt("CODEX_DATABASE_POSTGRES_HOST") - .unwrap_or_else(|| "localhost".to_string()), - port: env_or("CODEX_DATABASE_POSTGRES_PORT", 5432), - username: env_string_opt("CODEX_DATABASE_POSTGRES_USERNAME") - .unwrap_or_else(|| "codex".to_string()), - password: env_string_opt("CODEX_DATABASE_POSTGRES_PASSWORD") - .unwrap_or_else(|| "codex".to_string()), - database_name: env_string_opt("CODEX_DATABASE_POSTGRES_DATABASE_NAME") - .unwrap_or_else(|| "codex".to_string()), + host: "localhost".to_string(), + port: 5432, + username: "codex".to_string(), + password: "codex".to_string(), + database_name: "codex".to_string(), // Pool settings - PostgreSQL can handle more concurrent connections - max_connections: env_or("CODEX_DATABASE_POSTGRES_MAX_CONNECTIONS", 25), - min_connections: env_or("CODEX_DATABASE_POSTGRES_MIN_CONNECTIONS", 2), - acquire_timeout_seconds: env_or("CODEX_DATABASE_POSTGRES_ACQUIRE_TIMEOUT", 30), - idle_timeout_seconds: env_or("CODEX_DATABASE_POSTGRES_IDLE_TIMEOUT", 600), - max_lifetime_seconds: env_or("CODEX_DATABASE_POSTGRES_MAX_LIFETIME", 3600), - operation_deadline_seconds: env_or("CODEX_DATABASE_POSTGRES_OPERATION_DEADLINE", 30), - batch_fan_out: env_or("CODEX_DATABASE_POSTGRES_BATCH_FAN_OUT", 8), - background_max_connections: env_or( - "CODEX_DATABASE_POSTGRES_BACKGROUND_MAX_CONNECTIONS", - 16, - ), + max_connections: 25, + min_connections: 2, + acquire_timeout_seconds: 30, + idle_timeout_seconds: 600, + max_lifetime_seconds: 3600, + operation_deadline_seconds: 30, + batch_fan_out: 8, + background_max_connections: 16, } } } @@ -760,21 +692,17 @@ impl Default for SQLiteConfig { pragmas.insert("journal_mode".to_string(), "WAL".to_string()); Self { - path: env_string_opt("CODEX_DATABASE_SQLITE_PATH") - .unwrap_or_else(|| "data/codex.db".to_string()), + path: "data/codex.db".to_string(), pragmas: Some(pragmas), // Pool settings - SQLite is more conservative due to single-writer lock - max_connections: env_or("CODEX_DATABASE_SQLITE_MAX_CONNECTIONS", 64), - min_connections: env_or("CODEX_DATABASE_SQLITE_MIN_CONNECTIONS", 2), - acquire_timeout_seconds: env_or("CODEX_DATABASE_SQLITE_ACQUIRE_TIMEOUT", 30), - idle_timeout_seconds: env_or("CODEX_DATABASE_SQLITE_IDLE_TIMEOUT", 300), - max_lifetime_seconds: env_or("CODEX_DATABASE_SQLITE_MAX_LIFETIME", 1800), - operation_deadline_seconds: env_or("CODEX_DATABASE_SQLITE_OPERATION_DEADLINE", 30), - batch_fan_out: env_or("CODEX_DATABASE_SQLITE_BATCH_FAN_OUT", 4), - background_max_connections: env_or( - "CODEX_DATABASE_SQLITE_BACKGROUND_MAX_CONNECTIONS", - 4, - ), + max_connections: 64, + min_connections: 2, + acquire_timeout_seconds: 30, + idle_timeout_seconds: 300, + max_lifetime_seconds: 1800, + operation_deadline_seconds: 30, + batch_fan_out: 4, + background_max_connections: 4, } } } @@ -805,9 +733,9 @@ impl ApplicationConfig { impl Default for ApplicationConfig { fn default() -> Self { Self { - host: env_string_opt("CODEX_APPLICATION_HOST").unwrap_or_else(|| "0.0.0.0".to_string()), - port: env_or("CODEX_APPLICATION_PORT", 8080), - base_url: env_string_opt("CODEX_APPLICATION_BASE_URL"), + host: "0.0.0.0".to_string(), + port: 8080, + base_url: None, } } } @@ -823,18 +751,9 @@ pub struct LoggingConfig { impl Default for LoggingConfig { fn default() -> Self { Self { - level: env_string_opt("CODEX_LOGGING_LEVEL") - .and_then(|s| match s.to_lowercase().as_str() { - "error" => Some(LogLevel::Error), - "warn" => Some(LogLevel::Warn), - "info" => Some(LogLevel::Info), - "debug" => Some(LogLevel::Debug), - "trace" => Some(LogLevel::Trace), - _ => None, - }) - .unwrap_or(LogLevel::Info), - console: env_bool_or("CODEX_LOGGING_CONSOLE", true), - file: env_string_opt("CODEX_LOGGING_FILE"), + level: LogLevel::Info, + console: true, + file: None, } } } @@ -886,7 +805,7 @@ pub struct ScannerConfig { impl Default for ScannerConfig { fn default() -> Self { Self { - max_concurrent_scans: env_or("CODEX_SCANNER_MAX_CONCURRENT_SCANS", 2), + max_concurrent_scans: 2, } } } @@ -905,8 +824,7 @@ pub struct SchedulerConfig { impl Default for SchedulerConfig { fn default() -> Self { Self { - timezone: env_string_opt("CODEX_SCHEDULER_TIMEZONE") - .unwrap_or_else(|| "UTC".to_string()), + timezone: "UTC".to_string(), } } } @@ -931,12 +849,9 @@ pub struct FilesConfig { impl Default for FilesConfig { fn default() -> Self { Self { - thumbnail_dir: env_string_opt("CODEX_FILES_THUMBNAIL_DIR") - .unwrap_or_else(|| "data/thumbnails".to_string()), - uploads_dir: env_string_opt("CODEX_FILES_UPLOADS_DIR") - .unwrap_or_else(|| "data/uploads".to_string()), - plugins_dir: env_string_opt("CODEX_FILES_PLUGINS_DIR") - .unwrap_or_else(|| "data/plugins".to_string()), + thumbnail_dir: "data/thumbnails".to_string(), + uploads_dir: "data/uploads".to_string(), + plugins_dir: "data/plugins".to_string(), } } } @@ -982,12 +897,11 @@ pub struct PdfConfig { impl Default for PdfConfig { fn default() -> Self { Self { - pdfium_library_path: env_string_opt("CODEX_PDF_PDFIUM_LIBRARY_PATH"), - render_dpi: env_or("CODEX_PDF_RENDER_DPI", 150), - jpeg_quality: env_or("CODEX_PDF_JPEG_QUALITY", 85), - cache_rendered_pages: env_bool_or("CODEX_PDF_CACHE_RENDERED_PAGES", true), - cache_dir: env_string_opt("CODEX_PDF_CACHE_DIR") - .unwrap_or_else(|| "data/cache".to_string()), + pdfium_library_path: None, + render_dpi: 150, + jpeg_quality: 85, + cache_rendered_pages: true, + cache_dir: "data/cache".to_string(), } } } @@ -1019,10 +933,10 @@ pub struct PdfHandleCacheConfig { impl Default for PdfHandleCacheConfig { fn default() -> Self { Self { - enabled: env_bool_or("CODEX_PDF_HANDLE_CACHE_ENABLED", true), - capacity: env_or("CODEX_PDF_HANDLE_CACHE_CAPACITY", 256), - idle_ttl_minutes: env_or("CODEX_PDF_HANDLE_CACHE_IDLE_TTL_MINUTES", 15), - sweep_interval_seconds: env_or("CODEX_PDF_HANDLE_CACHE_SWEEP_INTERVAL_SECONDS", 60), + enabled: true, + capacity: 256, + idle_ttl_minutes: 15, + sweep_interval_seconds: 60, } } } @@ -1030,20 +944,14 @@ impl Default for PdfHandleCacheConfig { impl Default for EmailConfig { fn default() -> Self { Self { - smtp_host: env_string_opt("CODEX_EMAIL_SMTP_HOST") - .unwrap_or_else(|| "localhost".to_string()), - smtp_port: env_or("CODEX_EMAIL_SMTP_PORT", 587), - smtp_username: env_string_opt("CODEX_EMAIL_SMTP_USERNAME").unwrap_or_default(), - smtp_password: env_string_opt("CODEX_EMAIL_SMTP_PASSWORD").unwrap_or_default(), - smtp_from_email: env_string_opt("CODEX_EMAIL_SMTP_FROM_EMAIL") - .unwrap_or_else(|| "noreply@example.com".to_string()), - smtp_from_name: env_string_opt("CODEX_EMAIL_SMTP_FROM_NAME") - .unwrap_or_else(|| "Codex".to_string()), - verification_token_expiry_hours: env_or( - "CODEX_EMAIL_VERIFICATION_TOKEN_EXPIRY_HOURS", - 24, - ), - verification_url_base: env_string_opt("CODEX_EMAIL_VERIFICATION_URL_BASE"), + smtp_host: "localhost".to_string(), + smtp_port: 587, + smtp_username: String::new(), + smtp_password: String::new(), + smtp_from_email: "noreply@example.com".to_string(), + smtp_from_name: "Codex".to_string(), + verification_token_expiry_hours: 24, + verification_url_base: None, } } } @@ -1059,9 +967,14 @@ impl Default for EmailConfig { pub enum OtlpProtocol { #[default] Grpc, - #[serde(alias = "http-protobuf", alias = "http_protobuf", alias = "httpproto")] + #[serde( + alias = "http/protobuf", + alias = "http-protobuf", + alias = "http_protobuf", + alias = "httpproto" + )] HttpProtobuf, - #[serde(alias = "http-json", alias = "http_json")] + #[serde(alias = "http/json", alias = "http-json", alias = "http_json")] HttpJson, } @@ -1124,20 +1037,11 @@ impl OtlpConfig { impl Default for OtlpConfig { fn default() -> Self { Self { - endpoint: env_string_opt("CODEX_OBSERVABILITY_OTLP_ENDPOINT").unwrap_or_default(), - protocol: env_string_opt("CODEX_OBSERVABILITY_OTLP_PROTOCOL") - .and_then(|s| match s.to_lowercase().as_str() { - "grpc" => Some(OtlpProtocol::Grpc), - "http/protobuf" | "http-protobuf" | "http_protobuf" | "httpproto" => { - Some(OtlpProtocol::HttpProtobuf) - } - "http/json" | "http-json" | "http_json" => Some(OtlpProtocol::HttpJson), - _ => None, - }) - .unwrap_or_default(), + endpoint: String::new(), + protocol: OtlpProtocol::default(), headers: HashMap::new(), - timeout_ms: env_or("CODEX_OBSERVABILITY_OTLP_TIMEOUT_MS", 5000), - proxy_endpoint: env_string_opt("CODEX_OBSERVABILITY_OTLP_PROXY_ENDPOINT"), + timeout_ms: 5000, + proxy_endpoint: None, } } } @@ -1158,8 +1062,8 @@ pub struct ObservabilityTracesConfig { impl Default for ObservabilityTracesConfig { fn default() -> Self { Self { - enabled: env_bool_or("CODEX_OBSERVABILITY_TRACES_ENABLED", true), - sample_ratio: env_or("CODEX_OBSERVABILITY_TRACES_SAMPLE_RATIO", 1.0_f64), + enabled: true, + sample_ratio: 1.0_f64, } } } @@ -1179,8 +1083,8 @@ pub struct ObservabilityMetricsConfig { impl Default for ObservabilityMetricsConfig { fn default() -> Self { Self { - enabled: env_bool_or("CODEX_OBSERVABILITY_METRICS_ENABLED", true), - export_interval_ms: env_or("CODEX_OBSERVABILITY_METRICS_EXPORT_INTERVAL_MS", 30000), + enabled: true, + export_interval_ms: 30000, } } } @@ -1208,10 +1112,9 @@ pub struct ObservabilityBrowserConfig { impl Default for ObservabilityBrowserConfig { fn default() -> Self { Self { - enabled: env_bool_or("CODEX_OBSERVABILITY_BROWSER_ENABLED", false), - proxy_path: env_string_opt("CODEX_OBSERVABILITY_BROWSER_PROXY_PATH") - .unwrap_or_else(|| "/api/v1/observability/otlp".to_string()), - sample_ratio: env_or("CODEX_OBSERVABILITY_BROWSER_SAMPLE_RATIO", 0.1_f64), + enabled: false, + proxy_path: "/api/v1/observability/otlp".to_string(), + sample_ratio: 0.1_f64, } } } @@ -1247,9 +1150,8 @@ pub struct ObservabilityConfig { impl Default for ObservabilityConfig { fn default() -> Self { Self { - enabled: env_bool_or("CODEX_OBSERVABILITY_ENABLED", false), - service_name: env_string_opt("CODEX_OBSERVABILITY_SERVICE_NAME") - .unwrap_or_else(|| "codex".to_string()), + enabled: false, + service_name: "codex".to_string(), otlp: OtlpConfig::default(), traces: ObservabilityTracesConfig::default(), metrics: ObservabilityMetricsConfig::default(), @@ -1261,6 +1163,12 @@ impl Default for ObservabilityConfig { #[cfg(test)] mod tests { use super::*; + + // The `resolve_data_dir` unit tests that lived here are gone with the + // method. Rooting paths under `data_dir` now depends on which layer + // supplied each key, so it can only be exercised through the real loader: + // see the path-rooting tests in `loader.rs`. + use serial_test::serial; #[test] @@ -2499,97 +2407,6 @@ database: assert_eq!(config.data_dir, "/var/lib/codex"); } - #[test] - fn test_resolve_data_dir_replaces_defaults() { - let mut config = Config { - data_dir: "/var/lib/codex".to_string(), - ..Config::default() - }; - // Set old defaults - config.files.thumbnail_dir = "data/thumbnails".to_string(); - config.files.uploads_dir = "data/uploads".to_string(); - config.files.plugins_dir = "data/plugins".to_string(); - config.pdf.cache_dir = "data/cache".to_string(); - config.database.sqlite = Some(SQLiteConfig { - path: "data/codex.db".to_string(), - pragmas: None, - ..SQLiteConfig::default() - }); - - config.resolve_data_dir(); - - assert_eq!(config.files.thumbnail_dir, "/var/lib/codex/thumbnails"); - assert_eq!(config.files.uploads_dir, "/var/lib/codex/uploads"); - assert_eq!(config.files.plugins_dir, "/var/lib/codex/plugins"); - assert_eq!(config.pdf.cache_dir, "/var/lib/codex/cache"); - assert_eq!( - config.database.sqlite.as_ref().unwrap().path, - "/var/lib/codex/codex.db" - ); - } - - #[test] - fn test_resolve_data_dir_preserves_explicit_overrides() { - let mut config = Config { - data_dir: "/var/lib/codex".to_string(), - ..Config::default() - }; - // Set custom (non-default) paths that should be preserved - config.files.thumbnail_dir = "/custom/thumbs".to_string(); - config.files.uploads_dir = "/custom/uploads".to_string(); - config.files.plugins_dir = "/custom/plugins".to_string(); - config.pdf.cache_dir = "/custom/cache".to_string(); - config.database.sqlite = Some(SQLiteConfig { - path: "/custom/db.sqlite".to_string(), - pragmas: None, - ..SQLiteConfig::default() - }); - - config.resolve_data_dir(); - - // Non-default paths should NOT be replaced - assert_eq!(config.files.thumbnail_dir, "/custom/thumbs"); - assert_eq!(config.files.uploads_dir, "/custom/uploads"); - assert_eq!(config.files.plugins_dir, "/custom/plugins"); - assert_eq!(config.pdf.cache_dir, "/custom/cache"); - assert_eq!( - config.database.sqlite.as_ref().unwrap().path, - "/custom/db.sqlite" - ); - } - - #[test] - fn test_resolve_data_dir_noop_with_default_data_dir() { - let mut config = Config::default(); - // data_dir is "data" by default, so old defaults like "data/thumbnails" - // should remain "data/thumbnails" - let original_thumb = config.files.thumbnail_dir.clone(); - let original_uploads = config.files.uploads_dir.clone(); - let original_plugins = config.files.plugins_dir.clone(); - let original_cache = config.pdf.cache_dir.clone(); - - config.resolve_data_dir(); - - assert_eq!(config.files.thumbnail_dir, original_thumb); - assert_eq!(config.files.uploads_dir, original_uploads); - assert_eq!(config.files.plugins_dir, original_plugins); - assert_eq!(config.pdf.cache_dir, original_cache); - } - - #[test] - fn test_resolve_data_dir_no_sqlite_config() { - let mut config = Config { - data_dir: "/var/lib/codex".to_string(), - ..Config::default() - }; - config.database.sqlite = None; - - // Should not panic when sqlite is None - config.resolve_data_dir(); - - assert_eq!(config.files.thumbnail_dir, "/var/lib/codex/thumbnails"); - } - #[test] fn test_plugins_dir_in_files_config() { let yaml_content = r#" @@ -2741,3 +2558,56 @@ observability: assert_eq!(cfg.effective_proxy_endpoint(), "http://collector:4318"); } } + +#[cfg(test)] +mod default_snapshot { + use super::*; + + /// Byte-for-byte record of `Config::default()`, captured before the + /// environment reads were lifted out of the `Default` impls. + /// + /// Those impls used to call `env_or` / `env_bool_or` / `env_string_opt`, + /// which meant the "defaults" layer silently depended on the environment. + /// Moving to a single figment precedence chain required turning ~40 of + /// those into literals, and a mistyped literal there is invisible: the + /// config still loads, just with a different number in it. This snapshot + /// is what makes that class of slip loud. + /// + /// To regenerate deliberately, print `serde_json::to_string_pretty` of + /// `Config::default()` and overwrite the file. + const GOLDEN: &str = include_str!("testdata/default_config.json"); + + #[test] + fn defaults_match_the_recorded_snapshot() { + let expected: serde_json::Value = + serde_json::from_str(GOLDEN).expect("golden snapshot should be valid JSON"); + let actual = serde_json::to_value(Config::default()).unwrap(); + + assert_eq!( + actual, expected, + "Config::default() drifted from the recorded snapshot. If the change \ + is intentional, regenerate src/testdata/default_config.json; if not, \ + a default was changed by accident." + ); + } + + /// The defaults layer must not read the environment. Anything that does + /// belongs in the figment env provider, which sits above the file layer; + /// a `Default` impl that reads env would sit *below* it and would win or + /// lose depending on whether the key also appears in the config file. + #[test] + fn defaults_do_not_read_the_environment() { + let source = include_str!("types.rs"); + let body = source + .split_once("mod default_snapshot") + .map(|(before, _)| before) + .unwrap_or(source); + + for needle in ["env_or(", "env_bool_or(", "env::var("] { + assert!( + !body.contains(needle), + "`{needle}` reappeared in types.rs; defaults must stay environment-independent" + ); + } + } +} diff --git a/src/commands/seed.rs b/src/commands/seed.rs index 43ff627ff..f48272b50 100644 --- a/src/commands/seed.rs +++ b/src/commands/seed.rs @@ -3,7 +3,7 @@ use chrono::Utc; use codex_api::permissions::{ ADMIN_PERMISSIONS, MAINTAINER_PERMISSIONS, READER_PERMISSIONS, serialize_permissions, }; -use codex_config::{Config, EnvOverride}; +use codex_config::Config; use codex_db::Database; use codex_db::entities::{api_keys, plugins::PluginPermission, users}; use codex_db::repositories::{ @@ -155,8 +155,7 @@ pub async fn seed_command(config_path: PathBuf, seed_config_path: Option Date: Sat, 15 Aug 2026 15:36:51 -0700 Subject: [PATCH 02/11] feat(config): accept the environment value shapes the override layer allowed Environment variables are strings, but the config tree holds bools, lists and maps. The deleted override layer hand-parsed each of those at its point of use, so plain serde would now reject input that used to work. These deserializers keep the accepted spellings: - bools take `true`/`false`, `1`/`0`, `yes`/`no` and `on`/`off`, in any case, which matters because figment parses `true` into a bool and `1` into a number before serde sees either - string lists take a YAML sequence or a comma-separated string, so `CODEX_API__CORS_ORIGINS=https://a,https://b` works - OTLP headers take a mapping or `k1=v1,k2=v2`, the form the whole header set is conventionally supplied in - OIDC role mapping takes one role per variable with a comma-separated group list, matching how it was set before - an empty optional string means absent, because blanking a variable and unsetting it are the same gesture in most deployment tooling and `env_string_opt` filtered empties Unrecognized input is now an error. The override layer used `if let Ok(v) = s.parse()`, so `CODEX_KOMGA_API_ENABLED=ture` quietly meant `false` and nothing said so. A typo in a deployment's environment should stop the process rather than flip a setting behind the operator's back. `OidcProviderConfig` gains struct-level defaults so a provider can again be introduced entirely from the environment, which the override layer supported and a missing-field parse error had broken. A provider that exists but is unusable is better reported by config validation, which names the actual problem, than by a parse error about `display_name`. --- crates/codex-config/src/de.rs | 387 ++++++++++++++++++++++++++++++ crates/codex-config/src/lib.rs | 1 + crates/codex-config/src/loader.rs | 149 ++++++++++++ crates/codex-config/src/types.rs | 56 +++++ 4 files changed, 593 insertions(+) create mode 100644 crates/codex-config/src/de.rs diff --git a/crates/codex-config/src/de.rs b/crates/codex-config/src/de.rs new file mode 100644 index 000000000..0c1172aba --- /dev/null +++ b/crates/codex-config/src/de.rs @@ -0,0 +1,387 @@ +//! Deserializers that accept the input shapes operators already use. +//! +//! Environment variables are strings, but the config tree holds bools, lists +//! and maps. The v1 override layer hand-parsed each of those at the point of +//! use; plain serde would now reject the same input. These deserializers keep +//! the accepted spellings working through the figment env provider. +//! +//! One deliberate behaviour change: v1 swallowed parse failures +//! (`if let Ok(v) = s.parse()`), so `CODEX_KOMGA_API_ENABLED=ture` silently +//! meant `false`. These reject unrecognized input instead. A typo in a +//! deployment's environment should stop the process, not quietly flip a +//! setting. + +use serde::de::{self, MapAccess, SeqAccess, Visitor}; +use serde::{Deserialize, Deserializer}; +use std::collections::HashMap; +use std::fmt; + +/// A bool from a bool, a number, or a string. +/// +/// figment parses `true` into a bool and `1` into a number before serde sees +/// them, so a field that must accept both has to handle both. +pub fn truthy_bool<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + struct TruthyBool; + + impl<'de> Visitor<'de> for TruthyBool { + type Value = bool; + + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("a boolean: true/false, 1/0, yes/no or on/off") + } + + fn visit_bool(self, value: bool) -> Result { + Ok(value) + } + + fn visit_u64(self, value: u64) -> Result { + match value { + 0 => Ok(false), + 1 => Ok(true), + other => Err(E::custom(format!("expected 0 or 1, found {other}"))), + } + } + + fn visit_i64(self, value: i64) -> Result { + match value { + 0 => Ok(false), + 1 => Ok(true), + other => Err(E::custom(format!("expected 0 or 1, found {other}"))), + } + } + + fn visit_str(self, value: &str) -> Result { + match value.trim().to_ascii_lowercase().as_str() { + "true" | "1" | "yes" | "on" => Ok(true), + "false" | "0" | "no" | "off" => Ok(false), + other => Err(E::custom(format!( + "expected true/false, 1/0, yes/no or on/off, found `{other}`" + ))), + } + } + } + + deserializer.deserialize_any(TruthyBool) +} + +/// A list of strings from a real list or a comma-separated string. +/// +/// `CODEX_API__CORS_ORIGINS=https://a.example,https://b.example` has to mean +/// the same thing as a YAML sequence. +pub fn string_list<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + struct StringList; + + impl<'de> Visitor<'de> for StringList { + type Value = Vec; + + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("a list of strings, or a comma-separated string") + } + + fn visit_str(self, value: &str) -> Result, E> { + Ok(split_csv(value)) + } + + fn visit_seq>(self, mut seq: A) -> Result, A::Error> { + let mut out = Vec::new(); + while let Some(item) = seq.next_element::()? { + out.push(item); + } + Ok(out) + } + } + + deserializer.deserialize_any(StringList) +} + +/// A string map from a real map or a `k1=v1,k2=v2` string. +/// +/// Used for OTLP headers, where the whole set is conventionally supplied in +/// one variable. +pub fn string_map<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + struct StringMap; + + impl<'de> Visitor<'de> for StringMap { + type Value = HashMap; + + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("a map, or a comma-separated list of key=value pairs") + } + + fn visit_str(self, value: &str) -> Result { + let mut out = HashMap::new(); + for entry in split_csv(value) { + let (key, val) = entry + .split_once('=') + .ok_or_else(|| E::custom(format!("expected `key=value`, found `{entry}`")))?; + let key = key.trim(); + if key.is_empty() { + return Err(E::custom(format!("empty key in `{entry}`"))); + } + out.insert(key.to_string(), val.trim().to_string()); + } + Ok(out) + } + + fn visit_map>(self, mut map: A) -> Result { + let mut out = HashMap::new(); + while let Some((key, value)) = map.next_entry::()? { + out.insert(key, value); + } + Ok(out) + } + } + + deserializer.deserialize_any(StringMap) +} + +/// A map whose values are themselves comma-separated lists. +/// +/// OIDC role mapping is set one role at a time +/// (`CODEX_AUTH__OIDC__PROVIDERS__X__ROLE_MAPPING__ADMIN=grp-a,grp-b`), so the +/// values arrive as strings rather than sequences. +pub fn string_list_map<'de, D>(deserializer: D) -> Result>, D::Error> +where + D: Deserializer<'de>, +{ + struct StringListMap; + + impl<'de> Visitor<'de> for StringListMap { + type Value = HashMap>; + + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("a map of names to string lists") + } + + fn visit_map>(self, mut map: A) -> Result { + #[derive(Deserialize)] + struct Entry(#[serde(deserialize_with = "string_list")] Vec); + + let mut out = HashMap::new(); + while let Some((key, Entry(value))) = map.next_entry::()? { + out.insert(key, value); + } + Ok(out) + } + } + + deserializer.deserialize_map(StringListMap) +} + +/// An optional string where empty means absent. +/// +/// v1's `env_string_opt` filtered empty values, so `CODEX_LOGGING__FILE=` +/// meant "no log file" rather than "log to a file named empty string". Keeping +/// that matters because unsetting a variable and blanking it are the same +/// gesture in most deployment tooling. +pub fn optional_string<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + struct OptionalString; + + impl<'de> Visitor<'de> for OptionalString { + type Value = Option; + + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("a string, or nothing") + } + + fn visit_none(self) -> Result { + Ok(None) + } + + fn visit_unit(self) -> Result { + Ok(None) + } + + fn visit_str(self, value: &str) -> Result { + Ok(non_empty(value)) + } + + fn visit_some>(self, inner: D2) -> Result { + let value = String::deserialize(inner)?; + Ok(non_empty(&value)) + } + } + + deserializer.deserialize_option(OptionalString) +} + +fn non_empty(value: &str) -> Option { + (!value.is_empty()).then(|| value.to_string()) +} + +/// Split on commas, trimming each entry and dropping empties, so a trailing +/// comma or padded list does not produce blank members. +fn split_csv(value: &str) -> Vec { + value + .split(',') + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(str::to_string) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use serde::Deserialize; + + #[derive(Debug, Deserialize)] + struct BoolHolder { + #[serde(deserialize_with = "truthy_bool")] + value: bool, + } + + #[derive(Debug, Deserialize)] + struct ListHolder { + #[serde(deserialize_with = "string_list")] + value: Vec, + } + + #[derive(Debug, Deserialize)] + struct MapHolder { + #[serde(deserialize_with = "string_map")] + value: HashMap, + } + + #[derive(Debug, Deserialize)] + struct OptHolder { + #[serde(deserialize_with = "optional_string")] + value: Option, + } + + fn parse_bool(yaml: &str) -> Result { + serde_yaml::from_str::(yaml).map(|h| h.value) + } + + #[test] + fn bools_accept_the_spellings_v1_accepted() { + for yaml in ["value: true", "value: 'true'", "value: 'TRUE'", "value: 1"] { + assert!(parse_bool(yaml).unwrap(), "{yaml} should be true"); + } + for yaml in [ + "value: false", + "value: 'false'", + "value: 'False'", + "value: 0", + ] { + assert!(!parse_bool(yaml).unwrap(), "{yaml} should be false"); + } + } + + #[test] + fn bools_accept_yes_no_and_on_off() { + assert!(parse_bool("value: 'yes'").unwrap()); + assert!(parse_bool("value: 'on'").unwrap()); + assert!(!parse_bool("value: 'no'").unwrap()); + assert!(!parse_bool("value: 'off'").unwrap()); + } + + /// v1 treated anything that was not `true`/`1` as `false`, so a typo + /// silently disabled a feature. This must fail instead. + #[test] + fn a_misspelled_bool_is_an_error_not_a_silent_false() { + let error = parse_bool("value: ture").unwrap_err().to_string(); + assert!( + error.contains("ture"), + "error should quote the input: {error}" + ); + + assert!( + parse_bool("value: 2").is_err(), + "only 0 and 1 are numeric bools" + ); + } + + #[test] + fn lists_accept_a_sequence_or_a_comma_separated_string() { + let from_seq = serde_yaml::from_str::("value:\n - a\n - b\n") + .unwrap() + .value; + let from_csv = serde_yaml::from_str::("value: 'a,b'") + .unwrap() + .value; + assert_eq!(from_seq, vec!["a", "b"]); + assert_eq!(from_csv, from_seq); + } + + #[test] + fn list_entries_are_trimmed_and_blanks_dropped() { + let parsed = serde_yaml::from_str::("value: ' a , b , '") + .unwrap() + .value; + assert_eq!(parsed, vec!["a", "b"]); + + let empty = serde_yaml::from_str::("value: ''") + .unwrap() + .value; + assert!(empty.is_empty()); + } + + #[test] + fn maps_accept_a_mapping_or_key_value_pairs() { + let from_map = serde_yaml::from_str::("value:\n a: '1'\n b: '2'\n") + .unwrap() + .value; + let from_pairs = serde_yaml::from_str::("value: 'a=1,b=2'") + .unwrap() + .value; + assert_eq!(from_map, from_pairs); + assert_eq!(from_pairs.get("a").map(String::as_str), Some("1")); + } + + #[test] + fn a_pair_without_an_equals_sign_is_an_error() { + let error = serde_yaml::from_str::("value: 'a=1,oops'") + .unwrap_err() + .to_string(); + assert!( + error.contains("oops"), + "error should quote the entry: {error}" + ); + } + + #[test] + fn a_value_may_itself_contain_an_equals_sign() { + let parsed = serde_yaml::from_str::("value: 'authorization=Bearer a=b'") + .unwrap() + .value; + assert_eq!( + parsed.get("authorization").map(String::as_str), + Some("Bearer a=b") + ); + } + + #[test] + fn an_empty_optional_string_is_absent() { + assert_eq!( + serde_yaml::from_str::("value: ''") + .unwrap() + .value, + None + ); + assert_eq!( + serde_yaml::from_str::("value: null") + .unwrap() + .value, + None + ); + assert_eq!( + serde_yaml::from_str::("value: /var/log/codex.log") + .unwrap() + .value, + Some("/var/log/codex.log".to_string()) + ); + } +} diff --git a/crates/codex-config/src/lib.rs b/crates/codex-config/src/lib.rs index 03e80c3fa..4c45d9019 100644 --- a/crates/codex-config/src/lib.rs +++ b/crates/codex-config/src/lib.rs @@ -3,6 +3,7 @@ //! Extracted from the monolithic `codex` crate as the first workspace leaf in //! the workspace-split plan. Has no dependencies on other Codex crates. +mod de; mod env_audit; mod keys; mod loader; diff --git a/crates/codex-config/src/loader.rs b/crates/codex-config/src/loader.rs index 928cb089c..b79bedd5f 100644 --- a/crates/codex-config/src/loader.rs +++ b/crates/codex-config/src/loader.rs @@ -487,6 +487,155 @@ auth: ); } + // ---- input shapes the v1 override layer accepted ---- + + #[test] + fn bools_accept_one_and_zero_from_the_environment() { + Jail::expect_with(|jail| { + jail.set_env("CODEX_KOMGA_API__ENABLED", "1"); + jail.set_env("CODEX_API__CORS_ENABLED", "0"); + + let config = Config::load(jail.directory().join("absent.yaml")).unwrap(); + + assert!(config.komga_api.enabled); + assert!(!config.api.cors_enabled); + Ok(()) + }); + } + + /// v1 read `eq_ignore_ascii_case("true") || == "1"`, so a typo meant + /// `false` and the operator never found out. It is now an error. + #[test] + fn a_misspelled_bool_stops_startup() { + Jail::expect_with(|jail| { + jail.set_env("CODEX_KOMGA_API__ENABLED", "ture"); + let error = Config::load(jail.directory().join("absent.yaml")).unwrap_err(); + assert!( + format!("{error:#}").contains("ture"), + "error should quote the bad value: {error:#}" + ); + Ok(()) + }); + } + + #[test] + fn comma_separated_lists_come_through_the_environment() { + Jail::expect_with(|jail| { + jail.set_env( + "CODEX_API__CORS_ORIGINS", + "https://a.example, https://b.example", + ); + jail.set_env("CODEX_RATE_LIMIT__EXEMPT_PATHS", "/health,/metrics"); + + let config = Config::load(jail.directory().join("absent.yaml")).unwrap(); + + assert_eq!( + config.api.cors_origins, + vec!["https://a.example", "https://b.example"] + ); + assert_eq!(config.rate_limit.exempt_paths, vec!["/health", "/metrics"]); + Ok(()) + }); + } + + #[test] + fn a_yaml_sequence_still_works_for_the_same_field() { + Jail::expect_with(|jail| { + jail.create_file( + "codex.yaml", + "api:\n cors_origins:\n - https://a.example\n - https://b.example\n", + )?; + let config = Config::load(jail.directory().join("codex.yaml")).unwrap(); + assert_eq!( + config.api.cors_origins, + vec!["https://a.example", "https://b.example"] + ); + Ok(()) + }); + } + + #[test] + fn otlp_headers_accept_key_value_pairs() { + Jail::expect_with(|jail| { + jail.set_env( + "CODEX_OBSERVABILITY__OTLP__HEADERS", + "authorization=Bearer tok,x-tenant=acme", + ); + + let config = Config::load(jail.directory().join("absent.yaml")).unwrap(); + + assert_eq!( + config + .observability + .otlp + .headers + .get("authorization") + .map(String::as_str), + Some("Bearer tok") + ); + assert_eq!( + config + .observability + .otlp + .headers + .get("x-tenant") + .map(String::as_str), + Some("acme") + ); + Ok(()) + }); + } + + /// Blanking a variable and unsetting it are the same gesture in most + /// deployment tooling, and v1's `env_string_opt` filtered empties. + #[test] + fn an_empty_optional_string_means_unset() { + Jail::expect_with(|jail| { + jail.create_file("codex.yaml", "logging:\n file: /var/log/codex.log\n")?; + jail.set_env("CODEX_LOGGING__FILE", ""); + + let config = Config::load(jail.directory().join("codex.yaml")).unwrap(); + + assert_eq!(config.logging.file, None); + Ok(()) + }); + } + + /// v1 set role mappings one role at a time; each value is a group list. + #[test] + fn oidc_role_mapping_is_set_per_role() { + Jail::expect_with(|jail| { + jail.set_env( + "CODEX_AUTH__OIDC__PROVIDERS__AUTHENTIK__ISSUER_URL", + "https://idp.example.com", + ); + jail.set_env( + "CODEX_AUTH__OIDC__PROVIDERS__AUTHENTIK__ROLE_MAPPING__ADMIN", + "codex-admins, platform", + ); + + let config = Config::load(jail.directory().join("absent.yaml")).unwrap(); + + let provider = &config.auth.oidc.providers["authentik"]; + assert_eq!(provider.issuer_url, "https://idp.example.com"); + assert_eq!( + provider.role_mapping.get("admin"), + Some(&vec!["codex-admins".to_string(), "platform".to_string()]) + ); + Ok(()) + }); + } + + #[test] + fn database_type_accepts_the_longer_spelling() { + Jail::expect_with(|jail| { + jail.set_env("CODEX_DATABASE__DB_TYPE", "postgresql"); + let config = Config::load(jail.directory().join("absent.yaml")).unwrap(); + assert_eq!(config.database.db_type, DatabaseType::Postgres); + Ok(()) + }); + } + #[test] fn overlay_path_gets_a_local_infix() { assert_eq!( diff --git a/crates/codex-config/src/types.rs b/crates/codex-config/src/types.rs index 487923ad1..2af2b34a4 100644 --- a/crates/codex-config/src/types.rs +++ b/crates/codex-config/src/types.rs @@ -17,6 +17,7 @@ pub struct TaskConfig { pub struct KomgaApiConfig { /// Enable Komga-compatible API endpoints /// When enabled, routes are mounted at /{prefix}/api/v1/* + #[serde(deserialize_with = "crate::de::truthy_bool")] pub enabled: bool, /// URL prefix for Komga API (default: "komga") @@ -36,6 +37,7 @@ fn default_komga_prefix() -> String { pub struct KoreaderApiConfig { /// Enable KOReader sync API endpoints /// When enabled, routes are mounted at /koreader/* + #[serde(deserialize_with = "crate::de::truthy_bool")] pub enabled: bool, } @@ -45,6 +47,7 @@ pub struct KoreaderApiConfig { #[serde(default)] pub struct RateLimitConfig { /// Enable rate limiting (default: true) + #[serde(deserialize_with = "crate::de::truthy_bool")] pub enabled: bool, /// Requests per second for anonymous users (default: 10) @@ -60,6 +63,7 @@ pub struct RateLimitConfig { pub authenticated_burst: u32, /// Glob patterns for paths exempt from rate limiting (e.g. `/api/v1/books/*/thumbnail`) + #[serde(deserialize_with = "crate::de::string_list")] pub exempt_paths: Vec, /// Cleanup interval in seconds for stale buckets (default: 60) @@ -99,6 +103,13 @@ impl OidcDefaultRole { /// Configuration for a single OIDC provider #[derive(Debug, Serialize, Deserialize, Clone)] +// Every field defaults, so a provider can be introduced entirely from the +// environment: setting `CODEX_AUTH__OIDC__PROVIDERS____ISSUER_URL` is +// enough to create the entry, matching what the v1 override layer allowed. +// A provider that is present but unusable (no issuer, no client id) is caught +// by config validation rather than by a missing-field parse error, so the +// message names the actual problem. +#[serde(default)] pub struct OidcProviderConfig { /// Display name shown on login button pub display_name: String, @@ -112,19 +123,23 @@ pub struct OidcProviderConfig { /// OAuth2 client secret (optional if using client_secret_env) #[serde(default, skip_serializing_if = "Option::is_none")] + #[serde(deserialize_with = "crate::de::optional_string")] pub client_secret: Option, /// Environment variable name containing client secret #[serde(default, skip_serializing_if = "Option::is_none")] + #[serde(deserialize_with = "crate::de::optional_string")] pub client_secret_env: Option, /// Scopes to request (openid is always included) #[serde(default)] + #[serde(deserialize_with = "crate::de::string_list")] pub scopes: Vec, /// Group-to-role mapping: role -> [group names] /// e.g., {"admin": ["codex-admins"], "reader": ["codex-users"]} #[serde(default)] + #[serde(deserialize_with = "crate::de::string_list_map")] pub role_mapping: HashMap>, /// Claim containing groups (default: "groups") @@ -145,9 +160,28 @@ pub struct OidcProviderConfig { /// client). Resolution of the empty default happens at validator /// construction. #[serde(default)] + #[serde(deserialize_with = "crate::de::string_list")] pub accepted_audiences: Vec, } +impl Default for OidcProviderConfig { + fn default() -> Self { + Self { + display_name: String::new(), + issuer_url: String::new(), + client_id: String::new(), + client_secret: None, + client_secret_env: None, + scopes: Vec::new(), + role_mapping: HashMap::new(), + groups_claim: default_groups_claim(), + username_claim: default_username_claim(), + email_claim: default_email_claim(), + accepted_audiences: Vec::new(), + } + } +} + fn default_groups_claim() -> String { "groups".to_string() } @@ -165,9 +199,11 @@ fn default_email_claim() -> String { #[serde(default)] pub struct OidcConfig { /// Enable OIDC authentication + #[serde(deserialize_with = "crate::de::truthy_bool")] pub enabled: bool, /// Auto-create users on first OIDC login + #[serde(deserialize_with = "crate::de::truthy_bool")] pub auto_create_users: bool, /// Default role for new OIDC users (if no group mapping matches) @@ -177,6 +213,7 @@ pub struct OidcConfig { /// e.g., "https://codex.example.com" or "http://localhost:8080" /// If not set, falls back to http://{application.host}:{application.port} #[serde(default, skip_serializing_if = "Option::is_none")] + #[serde(deserialize_with = "crate::de::optional_string")] pub redirect_uri_base: Option, /// Redirect targets a client may request at the end of an OIDC flow. @@ -190,6 +227,7 @@ pub struct OidcConfig { /// whatever target is used, so every entry here is a party trusted with the /// tokens of anyone who completes a sign-in. #[serde(default)] + #[serde(deserialize_with = "crate::de::string_list")] pub allowed_redirect_uris: Vec, /// Provider configurations keyed by provider name (e.g., "authentik", "keycloak") @@ -395,8 +433,10 @@ impl Default for Config { pub struct AuthConfig { pub jwt_secret: String, pub jwt_expiry_hours: u32, + #[serde(deserialize_with = "crate::de::truthy_bool")] pub refresh_token_enabled: bool, pub refresh_token_expiry_days: u32, + #[serde(deserialize_with = "crate::de::truthy_bool")] pub email_confirmation_required: bool, pub argon2_memory_cost: u32, pub argon2_time_cost: u32, @@ -426,9 +466,12 @@ impl Default for AuthConfig { #[serde(default)] pub struct ApiConfig { pub base_path: String, + #[serde(deserialize_with = "crate::de::truthy_bool")] pub enable_api_docs: bool, pub api_docs_path: String, + #[serde(deserialize_with = "crate::de::truthy_bool")] pub cors_enabled: bool, + #[serde(deserialize_with = "crate::de::string_list")] pub cors_origins: Vec, pub max_page_size: usize, } @@ -716,6 +759,7 @@ pub struct ApplicationConfig { /// Used as a fallback for OIDC redirect URIs and email verification links. /// If not set, falls back to http://{host}:{port} #[serde(default, skip_serializing_if = "Option::is_none")] + #[serde(deserialize_with = "crate::de::optional_string")] pub base_url: Option, } @@ -744,7 +788,9 @@ impl Default for ApplicationConfig { #[serde(default)] pub struct LoggingConfig { pub level: LogLevel, + #[serde(deserialize_with = "crate::de::truthy_bool")] pub console: bool, + #[serde(deserialize_with = "crate::de::optional_string")] pub file: Option, } @@ -869,6 +915,7 @@ pub struct EmailConfig { /// Base URL for email verification links (e.g., "https://codex.example.com") /// If not set, falls back to application.base_url, then http://{host}:{port} #[serde(default, skip_serializing_if = "Option::is_none")] + #[serde(deserialize_with = "crate::de::optional_string")] pub verification_url_base: Option, } @@ -878,6 +925,7 @@ pub struct EmailConfig { pub struct PdfConfig { /// Path to PDFium library (optional) /// If not set, will search current directory and system paths + #[serde(deserialize_with = "crate::de::optional_string")] pub pdfium_library_path: Option, /// Default render DPI (72-300, default 150) @@ -888,6 +936,7 @@ pub struct PdfConfig { pub jpeg_quality: u8, /// Enable rendered page caching (default: true) + #[serde(deserialize_with = "crate::de::truthy_bool")] pub cache_rendered_pages: bool, /// Directory for caching rendered PDF pages (default: data/cache) @@ -914,6 +963,7 @@ impl Default for PdfConfig { #[serde(default)] pub struct PdfHandleCacheConfig { /// Master switch. When false, every page render re-opens the PDF. + #[serde(deserialize_with = "crate::de::truthy_bool")] pub enabled: bool, /// Maximum number of open `PdfDocument` handles to keep resident. @@ -1007,6 +1057,7 @@ pub struct OtlpConfig { pub protocol: OtlpProtocol, /// Arbitrary headers attached to every export request (auth, tenancy). + #[serde(deserialize_with = "crate::de::string_map")] pub headers: HashMap, /// Per-export request timeout in milliseconds. @@ -1018,6 +1069,7 @@ pub struct OtlpConfig { /// the matching HTTP port (e.g. `:4318`) without changing the SDK /// transport. When unset or empty, the proxy falls back to `endpoint`. #[serde(default)] + #[serde(deserialize_with = "crate::de::optional_string")] pub proxy_endpoint: Option, } @@ -1052,6 +1104,7 @@ impl Default for OtlpConfig { pub struct ObservabilityTracesConfig { /// Enable trace export. Honored only when the parent `observability.enabled` /// is also true. + #[serde(deserialize_with = "crate::de::truthy_bool")] pub enabled: bool, /// Parent-based sampling ratio in `[0.0, 1.0]`. Values outside this range @@ -1074,6 +1127,7 @@ impl Default for ObservabilityTracesConfig { pub struct ObservabilityMetricsConfig { /// Enable metrics export. Honored only when the parent `observability.enabled` /// is also true. + #[serde(deserialize_with = "crate::de::truthy_bool")] pub enabled: bool, /// Periodic reader export interval in milliseconds. @@ -1099,6 +1153,7 @@ pub struct ObservabilityBrowserConfig { /// Independent of the backend `observability.enabled` flag because some /// operators want server-side observability without shipping spans from /// every browser tab. + #[serde(deserialize_with = "crate::de::truthy_bool")] pub enabled: bool, /// Path on the Codex server where the browser SDK POSTs OTLP batches. @@ -1128,6 +1183,7 @@ impl Default for ObservabilityBrowserConfig { #[serde(default)] pub struct ObservabilityConfig { /// Master switch. Must be `true` for any OTel work to happen. + #[serde(deserialize_with = "crate::de::truthy_bool")] pub enabled: bool, /// `service.name` resource attribute. Identifies this process in the From b11ff33ebcc9ee550bdd70e598589c724d29416e Mon Sep 17 00:00:00 2001 From: Sylvain Cau Date: Sat, 15 Aug 2026 15:53:49 -0700 Subject: [PATCH 03/11] fix(config): reject a configuration that names a database section it lacks `db_type: postgres` with no `database.postgres` section parsed cleanly and then panicked, because `display_database_config` unwrapped the missing section while logging startup state. The same held for `sqlite`. Both are now caught by a `Config::validate` that runs at the end of loading, so the process stops with a message naming the missing section and how to supply it instead of aborting inside a log statement. The logging function no longer unwraps at all. Reaching it without a section would mean a caller bypassed the loader, and logging is not the place to end the process. Validation also rejects an OIDC provider with no `issuer_url` or `client_id`, but only while OIDC is enabled, so a half-written provider left in a disabled block does not stop the server. This is the check that makes struct-level defaults on the provider safe: an entry can be created from the environment alone, and if it is unusable the error names the field that is missing rather than blaming `display_name`. --- crates/codex-cli-common/src/lib.rs | 13 +++++- crates/codex-config/src/lib.rs | 11 ++--- crates/codex-config/src/loader.rs | 72 ++++++++++++++++++++++++++++++ crates/codex-config/src/types.rs | 58 ++++++++++++++++++++++++ 4 files changed, 147 insertions(+), 7 deletions(-) diff --git a/crates/codex-cli-common/src/lib.rs b/crates/codex-cli-common/src/lib.rs index d7b20793f..90b96bb06 100644 --- a/crates/codex-cli-common/src/lib.rs +++ b/crates/codex-cli-common/src/lib.rs @@ -314,9 +314,15 @@ fn build_file_appender( /// Display database configuration pub fn display_database_config(config: &Config) { info!("Database Configuration:"); + // No unwrapping: a `db_type` whose section is missing is rejected by + // `Config::validate` during load, so reaching here without one would mean + // a caller bypassed the loader. Logging is not the place to abort. match config.database.db_type { DatabaseType::Postgres => { - let pg_config = config.database.postgres.as_ref().unwrap(); + let Some(pg_config) = config.database.postgres.as_ref() else { + warn!(" Type: PostgreSQL, but no `database.postgres` section is configured"); + return; + }; info!(" Type: PostgreSQL"); info!(" Host: {}", pg_config.host); info!(" Port: {}", pg_config.port); @@ -324,7 +330,10 @@ pub fn display_database_config(config: &Config) { info!(" Username: {}", pg_config.username); } DatabaseType::SQLite => { - let sqlite_config = config.database.sqlite.as_ref().unwrap(); + let Some(sqlite_config) = config.database.sqlite.as_ref() else { + warn!(" Type: SQLite, but no `database.sqlite` section is configured"); + return; + }; info!(" Type: SQLite"); info!(" Path: {}", sqlite_config.path); if let Some(pragmas) = &sqlite_config.pragmas { diff --git a/crates/codex-config/src/lib.rs b/crates/codex-config/src/lib.rs index 4c45d9019..4703c1cca 100644 --- a/crates/codex-config/src/lib.rs +++ b/crates/codex-config/src/lib.rs @@ -12,11 +12,12 @@ mod types; #[allow(unused_imports)] pub use types::{ - ApiConfig, ApplicationConfig, AuthConfig, Config, DatabaseConfig, DatabaseType, EmailConfig, - FilesConfig, KomgaApiConfig, KoreaderApiConfig, LoggingConfig, ObservabilityBrowserConfig, - ObservabilityConfig, ObservabilityMetricsConfig, ObservabilityTracesConfig, OidcConfig, - OidcDefaultRole, OidcProviderConfig, OtlpConfig, OtlpProtocol, PdfConfig, PdfHandleCacheConfig, - PostgresConfig, RateLimitConfig, SQLiteConfig, ScannerConfig, SchedulerConfig, TaskConfig, + ApiConfig, ApplicationConfig, AuthConfig, Config, ConfigError, DatabaseConfig, DatabaseType, + EmailConfig, FilesConfig, KomgaApiConfig, KoreaderApiConfig, LoggingConfig, + ObservabilityBrowserConfig, ObservabilityConfig, ObservabilityMetricsConfig, + ObservabilityTracesConfig, OidcConfig, OidcDefaultRole, OidcProviderConfig, OtlpConfig, + OtlpProtocol, PdfConfig, PdfHandleCacheConfig, PostgresConfig, RateLimitConfig, SQLiteConfig, + ScannerConfig, SchedulerConfig, TaskConfig, }; pub use env_audit::{ diff --git a/crates/codex-config/src/loader.rs b/crates/codex-config/src/loader.rs index b79bedd5f..3fd98fb60 100644 --- a/crates/codex-config/src/loader.rs +++ b/crates/codex-config/src/loader.rs @@ -50,6 +50,7 @@ impl Config { .with_context(|| format!("failed to parse configuration from {}", path.display()))?; config.root_paths_at_data_dir(&|key| overrides.find_value(key).is_ok()); + config.validate()?; Ok(config) } @@ -467,6 +468,10 @@ auth: let var = crate::v2_name_for(key); Jail::expect_with(|jail| { + // Keeps `db_type: postgres` valid when that is the key under + // test; harmless for every other key, since validation only + // inspects the section the active engine names. + jail.set_env("CODEX_DATABASE__POSTGRES__HOST", "probe.internal"); jail.set_env(&var, &raw); let config = Config::load(jail.directory().join("absent.yaml")) .unwrap_or_else(|e| panic!("{var}={raw} should load: {e:#}")); @@ -630,12 +635,79 @@ auth: fn database_type_accepts_the_longer_spelling() { Jail::expect_with(|jail| { jail.set_env("CODEX_DATABASE__DB_TYPE", "postgresql"); + jail.set_env("CODEX_DATABASE__POSTGRES__HOST", "db.internal"); let config = Config::load(jail.directory().join("absent.yaml")).unwrap(); assert_eq!(config.database.db_type, DatabaseType::Postgres); Ok(()) }); } + // ---- cross-field validation ---- + + /// Previously this parsed fine and then panicked in + /// `display_database_config`, which unwrapped the missing section. + #[test] + fn postgres_without_a_postgres_section_is_rejected_at_load() { + Jail::expect_with(|jail| { + jail.create_file("codex.yaml", "database:\n db_type: postgres\n")?; + + let error = Config::load(jail.directory().join("codex.yaml")).unwrap_err(); + let message = format!("{error:#}"); + + assert!( + message.contains("database.postgres"), + "error should name the missing section: {message}" + ); + Ok(()) + }); + } + + /// The same config becomes valid once the environment supplies the + /// section, which is the deployment shape this unblocks. + #[test] + fn postgres_from_the_environment_satisfies_validation() { + Jail::expect_with(|jail| { + jail.create_file("codex.yaml", "database:\n db_type: postgres\n")?; + jail.set_env("CODEX_DATABASE__POSTGRES__HOST", "db.internal"); + + let config = Config::load(jail.directory().join("codex.yaml")).unwrap(); + + assert_eq!(config.database.postgres.unwrap().host, "db.internal"); + Ok(()) + }); + } + + #[test] + fn an_oidc_provider_without_an_issuer_is_rejected_when_oidc_is_on() { + Jail::expect_with(|jail| { + jail.set_env("CODEX_AUTH__OIDC__ENABLED", "true"); + jail.set_env("CODEX_AUTH__OIDC__PROVIDERS__AUTHENTIK__CLIENT_ID", "codex"); + + let error = Config::load(jail.directory().join("absent.yaml")).unwrap_err(); + let message = format!("{error:#}"); + + assert!( + message.contains("authentik") && message.contains("issuer_url"), + "error should name the provider and the field: {message}" + ); + Ok(()) + }); + } + + /// A half-written provider in a disabled block must not stop the server. + #[test] + fn an_incomplete_provider_is_tolerated_while_oidc_is_off() { + Jail::expect_with(|jail| { + jail.set_env("CODEX_AUTH__OIDC__PROVIDERS__AUTHENTIK__CLIENT_ID", "codex"); + + let config = Config::load(jail.directory().join("absent.yaml")).unwrap(); + + assert!(!config.auth.oidc.enabled); + assert!(config.auth.oidc.providers.contains_key("authentik")); + Ok(()) + }); + } + #[test] fn overlay_path_gets_a_local_infix() { assert_eq!( diff --git a/crates/codex-config/src/types.rs b/crates/codex-config/src/types.rs index 2af2b34a4..8c48b8147 100644 --- a/crates/codex-config/src/types.rs +++ b/crates/codex-config/src/types.rs @@ -328,10 +328,68 @@ fn default_data_dir() -> String { const DEFAULT_THUMBNAILS_SUBDIR: &str = "thumbnails"; const DEFAULT_UPLOADS_SUBDIR: &str = "uploads"; const DEFAULT_PLUGINS_SUBDIR: &str = "plugins"; +/// A configuration that parsed but does not describe a usable system. +#[derive(Debug)] +pub struct ConfigError(String); + +impl std::fmt::Display for ConfigError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +impl std::error::Error for ConfigError {} + const DEFAULT_CACHE_SUBDIR: &str = "cache"; const DEFAULT_SQLITE_FILENAME: &str = "codex.db"; impl Config { + /// Cross-field checks that per-field deserialization cannot express. + /// + /// Called at the end of [`Config::load`], so a structurally valid but + /// semantically broken configuration fails at startup with a message about + /// the actual problem, rather than at the first request or the first cron + /// tick. + pub fn validate(&self) -> Result<(), ConfigError> { + match self.database.db_type { + DatabaseType::Postgres if self.database.postgres.is_none() => { + return Err(ConfigError( + "database.db_type is `postgres` but there is no \ + `database.postgres` section. Add one, or set at least \ + CODEX_DATABASE__POSTGRES__HOST." + .to_string(), + )); + } + DatabaseType::SQLite if self.database.sqlite.is_none() => { + return Err(ConfigError( + "database.db_type is `sqlite` but there is no \ + `database.sqlite` section." + .to_string(), + )); + } + _ => {} + } + + // Only checked when OIDC is on: a half-written provider left behind in + // a disabled block should not stop the server. + if self.auth.oidc.enabled { + for (name, provider) in &self.auth.oidc.providers { + if provider.issuer_url.trim().is_empty() { + return Err(ConfigError(format!( + "OIDC provider `{name}` has no issuer_url" + ))); + } + if provider.client_id.trim().is_empty() { + return Err(ConfigError(format!( + "OIDC provider `{name}` has no client_id" + ))); + } + } + } + + Ok(()) + } + /// Root the per-kind data directories under `data_dir`. /// /// Every sub-path (`files.*_dir`, `pdf.cache_dir`, the SQLite file) defaults From 306c14daad9f687b229c1836b7dc3c33905681ff Mon Sep 17 00:00:00 2001 From: Sylvain Cau Date: Sat, 15 Aug 2026 16:52:42 -0700 Subject: [PATCH 04/11] feat(config)!: make the ad-hoc environment reads real configuration keys Seven operator-tunable settings were read with `std::env::var` at their point of use rather than through `Config`, so they appeared in no config file, in no resolved-config dump, and in nothing that validates spelling. They are now ordinary keys: CODEX_COOKIE_SECURE -> auth.cookie_secure CODEX_DISABLE_WORKERS -> task.run_in_process (inverted) CODEX_IMAGE_DECODE_CONCURRENCY -> images.decode_concurrency CODEX_PLUGIN_ALLOWED_COMMANDS -> plugins.allowed_commands CODEX_SKIP_MIGRATIONS -> database.run_migrations (inverted) CODEX_MIGRATION_WAIT_TIMEOUT -> database.migration_wait_timeout_secs CODEX_MIGRATION_WAIT_INTERVAL -> database.migration_wait_interval_secs Both inverted keys default to the previous behaviour: workers run in process and migrations are applied unless something says otherwise. Three variables stay in the environment. The encryption key is read deep in codex-utils and codex-db, which have no configuration in scope, so moving it means threading config through those crates. The two `codex copy` database URLs are per-invocation arguments already backed by CLI flags. That leaves an allowlist of three, small enough that forgetting to update it is unlikely. Because two of these flip sense, a renamed variable cannot be derived by re-spelling the old name, and a deployment that kept `CODEX_DISABLE_WORKERS` and heard nothing would start task workers in a pod meant to serve web traffic only. A `REMOVED_VARS` table records each replacement and marks the inverted pair, so the validator can say what to set instead rather than guessing. The plugin allowlist keeps its process-wide `OnceLock`, now installed from config during startup instead of read from the environment on first use. `is_command_allowed` is reached from request handlers with no configuration in scope, and threading it down every call path would touch the whole plugin surface for a value fixed at boot. A second initialization is refused and logged rather than silently replacing the allowlist. Two `init_database` tests drove the old variables through `unsafe` env mutation and were caught by this change; they now set config fields and no longer need serialized execution. A third only checked that "1" was truthy, which the value deserializers already cover. --- .../codex-api/src/routes/v1/handlers/auth.rs | 29 ++- .../codex-api/src/routes/v1/handlers/oidc.rs | 1 + .../codex-api/src/routes/v1/handlers/setup.rs | 2 +- crates/codex-cli-common/src/lib.rs | 170 ++++++------------ crates/codex-config/src/de.rs | 102 ++++++++--- crates/codex-config/src/env_audit.rs | 152 ++++++++++++++-- crates/codex-config/src/keys.rs | 9 + crates/codex-config/src/lib.rs | 6 +- crates/codex-config/src/loader.rs | 73 ++++++++ .../src/testdata/default_config.json | 14 +- crates/codex-config/src/types.rs | 120 ++++++++++++- crates/codex-db/src/connection.rs | 5 + crates/codex-db/src/test_helpers.rs | 1 + crates/codex-migrate/src/conn.rs | 2 + crates/codex-services/src/plugin/process.rs | 59 +++--- crates/codex-services/src/refresh_token.rs | 1 + .../src/handlers/cleanup_refresh_tokens.rs | 1 + src/commands/serve.rs | 28 ++- src/commands/worker.rs | 14 +- tests/api/pool_contention.rs | 1 + tests/common/db.rs | 3 + tests/db/migrations.rs | 5 + tests/db/postgres.rs | 1 + tests/db/repositories.rs | 1 + 24 files changed, 569 insertions(+), 231 deletions(-) diff --git a/crates/codex-api/src/routes/v1/handlers/auth.rs b/crates/codex-api/src/routes/v1/handlers/auth.rs index c844e6007..cb22d039e 100644 --- a/crates/codex-api/src/routes/v1/handlers/auth.rs +++ b/crates/codex-api/src/routes/v1/handlers/auth.rs @@ -15,6 +15,7 @@ use axum::{ response::{IntoResponse, Response}, }; use chrono::Utc; +use codex_config::AuthConfig; use codex_db::{ entities::users, repositories::{EmailVerificationTokenRepository, SettingsRepository, UserRepository}, @@ -23,7 +24,6 @@ use codex_services::{RefreshTokenError, RefreshTokenService}; use codex_utils::password; use sea_orm::ActiveModelTrait; use sea_orm::Set; -use std::env; use std::sync::Arc; use uuid::Uuid; @@ -84,20 +84,13 @@ fn parse_permissions_json(json: &serde_json::Value) -> Vec { .unwrap_or_default() } -/// Build authentication cookie string +/// Build the authentication cookie. /// -/// Conditionally includes `Secure` flag based on environment: -/// - If `CODEX_COOKIE_SECURE` env var is set, uses that value -/// - Otherwise, defaults to `false` for development (allows HTTP cookies) -/// - In production, should be set to `true` via env var -pub(crate) fn build_auth_cookie(token: &str, max_age: u64) -> String { - // Check environment variable first - let use_secure = env::var("CODEX_COOKIE_SECURE") - .ok() - .and_then(|v| v.parse::().ok()) - .unwrap_or(false); // Default to false for development (allows HTTP) - - if use_secure { +/// `Secure` comes from `auth.cookie_secure`, which is off unless set so that +/// plain-HTTP development keeps working. Any deployment terminating TLS should +/// turn it on. +pub(crate) fn build_auth_cookie(auth: &AuthConfig, token: &str, max_age: u64) -> String { + if auth.cookie_secure() { format!( "auth_token={}; HttpOnly; Secure; SameSite=Lax; Path=/; Max-Age={}", token, max_age @@ -198,7 +191,7 @@ pub async fn login( // Create HTTP-only cookie for image authentication // Using SameSite=Lax instead of Strict to allow images to load from direct links - let cookie = build_auth_cookie(&access_token, 24 * 3600); + let cookie = build_auth_cookie(&state.auth_config, &access_token, 24 * 3600); // Build response with cookie let mut headers = HeaderMap::new(); @@ -355,7 +348,7 @@ pub async fn refresh( // Reissue the image-auth cookie so the new access token is in effect for // browser image requests as well as Authorization-header requests. - let cookie = build_auth_cookie(&access_token, 24 * 3600); + let cookie = build_auth_cookie(&state.auth_config, &access_token, 24 * 3600); let mut headers = HeaderMap::new(); headers.insert( header::SET_COOKIE, @@ -573,7 +566,7 @@ pub async fn register( }; // Create HTTP-only cookie for image authentication - let cookie = build_auth_cookie(&access_token, 24 * 3600); + let cookie = build_auth_cookie(&state.auth_config, &access_token, 24 * 3600); // Build response with cookie let mut headers = HeaderMap::new(); @@ -673,7 +666,7 @@ pub async fn verify_email( }; // Create HTTP-only cookie for image authentication - let cookie = build_auth_cookie(&access_token, 24 * 3600); + let cookie = build_auth_cookie(&state.auth_config, &access_token, 24 * 3600); // Build response with cookie let mut headers = HeaderMap::new(); diff --git a/crates/codex-api/src/routes/v1/handlers/oidc.rs b/crates/codex-api/src/routes/v1/handlers/oidc.rs index 09187a406..bcb9c5c92 100644 --- a/crates/codex-api/src/routes/v1/handlers/oidc.rs +++ b/crates/codex-api/src/routes/v1/handlers/oidc.rs @@ -484,6 +484,7 @@ pub async fn callback( // Create HTTP-only auth cookie (for image/resource requests) let cookie = build_auth_cookie( + &state.auth_config, &access_token, state.auth_config.jwt_expiry_hours as u64 * 3600, ); diff --git a/crates/codex-api/src/routes/v1/handlers/setup.rs b/crates/codex-api/src/routes/v1/handlers/setup.rs index 9fa6f103b..a51ac1e19 100644 --- a/crates/codex-api/src/routes/v1/handlers/setup.rs +++ b/crates/codex-api/src/routes/v1/handlers/setup.rs @@ -216,7 +216,7 @@ pub async fn initialize_setup( }; // Create HTTP-only cookie for image authentication (same as login) - let cookie = build_auth_cookie(&access_token, 24 * 3600); + let cookie = build_auth_cookie(&state.auth_config, &access_token, 24 * 3600); // Build response with cookie let mut headers = HeaderMap::new(); diff --git a/crates/codex-cli-common/src/lib.rs b/crates/codex-cli-common/src/lib.rs index 90b96bb06..f001a627c 100644 --- a/crates/codex-cli-common/src/lib.rs +++ b/crates/codex-cli-common/src/lib.rs @@ -359,24 +359,18 @@ struct DbWait { } impl DbWait { - /// Environment variables (used when the parameter is None): - /// - CODEX_MIGRATION_WAIT_TIMEOUT: Timeout in seconds (default: 300) - /// - CODEX_MIGRATION_WAIT_INTERVAL: Check interval in seconds (default: 2) - fn resolve(timeout_seconds: Option, check_interval_seconds: Option) -> Self { - fn from_env(name: &str) -> Option { - std::env::var(name).ok().and_then(|v| v.parse().ok()) - } - + /// A CLI flag wins over the configured budget, which carries the default. + fn resolve( + config: &DatabaseConfig, + timeout_seconds: Option, + check_interval_seconds: Option, + ) -> Self { Self { timeout: Duration::from_secs( - timeout_seconds - .or_else(|| from_env("CODEX_MIGRATION_WAIT_TIMEOUT")) - .unwrap_or(300), // Default 5 minutes + timeout_seconds.unwrap_or(config.migration_wait_timeout_secs), ), check_interval: Duration::from_secs( - check_interval_seconds - .or_else(|| from_env("CODEX_MIGRATION_WAIT_INTERVAL")) - .unwrap_or(2), // Default 2 seconds + check_interval_seconds.unwrap_or(config.migration_wait_interval_secs), ), } } @@ -420,10 +414,11 @@ impl DbWait { /// could have asked instead. pub async fn wait_for_migrations_on( db: &Database, + config: &DatabaseConfig, timeout_seconds: Option, check_interval_seconds: Option, ) -> anyhow::Result<()> { - let wait = DbWait::resolve(timeout_seconds, check_interval_seconds); + let wait = DbWait::resolve(config, timeout_seconds, check_interval_seconds); wait.log("migrations to complete"); poll_until_migrated(db, &wait, std::time::Instant::now()).await } @@ -444,7 +439,7 @@ pub async fn connect_with_retry( timeout_seconds: Option, check_interval_seconds: Option, ) -> anyhow::Result { - let wait = DbWait::resolve(timeout_seconds, check_interval_seconds); + let wait = DbWait::resolve(config, timeout_seconds, check_interval_seconds); wait.log("the database to accept connections"); retry_connect(&wait, std::time::Instant::now(), || Database::new(config)).await } @@ -493,7 +488,7 @@ pub async fn wait_for_migrations_complete( timeout_seconds: Option, check_interval_seconds: Option, ) -> anyhow::Result<()> { - let wait = DbWait::resolve(timeout_seconds, check_interval_seconds); + let wait = DbWait::resolve(config, timeout_seconds, check_interval_seconds); wait.log("migrations to complete"); let start_time = std::time::Instant::now(); @@ -543,29 +538,24 @@ async fn poll_until_migrated( /// Initialize database connection and run migrations /// -/// If CODEX_SKIP_MIGRATIONS environment variable is set to "true" or "1", -/// migrations will be skipped and the function will wait for migrations to complete -/// (useful when migrations are run separately via a job/init container). +/// When `database.run_migrations` is false, this waits for the schema to be +/// current instead of applying it, which is what a deployment whose migrations +/// belong to a separate Job or init container needs. pub async fn init_database(config: &Config) -> anyhow::Result { info!("========================================"); info!("Initializing database connection..."); let db = Database::new(&config.database).await?; info!("Database connected successfully"); - // Check if migrations should be skipped - let skip_migrations = std::env::var("CODEX_SKIP_MIGRATIONS") - .map(|v| v.eq_ignore_ascii_case("true") || v == "1") - .unwrap_or(false); - - if skip_migrations { - info!("Skipping migrations (CODEX_SKIP_MIGRATIONS is set)"); + if !config.database.run_migrations { + info!("Skipping migrations (database.run_migrations is false)"); info!("Waiting for migrations to complete (run externally)..."); // Poll over the pool opened above rather than opening more. This process // is already connected; a wait that reconnects each time would spend the // whole timeout competing for connections with the deployment it is // waiting to join. // Timeout/interval come from the environment. - wait_for_migrations_on(&db, None, None).await?; + wait_for_migrations_on(&db, &config.database, None, None).await?; info!("Migrations are complete"); } else { // Run migrations to ensure database schema is up to date @@ -883,6 +873,7 @@ mod tests { pragmas: None, ..SQLiteConfig::default() }), + ..DatabaseConfig::default() } } @@ -975,7 +966,7 @@ mod tests { ); drop(reconnected); - let result = wait_for_migrations_on(&db, Some(2), Some(1)).await; + let result = wait_for_migrations_on(&db, &config, Some(2), Some(1)).await; assert!( result.is_ok(), @@ -994,7 +985,7 @@ mod tests { let db = Database::new(&config).await.unwrap(); let start = std::time::Instant::now(); - let result = wait_for_migrations_on(&db, Some(2), Some(1)).await; + let result = wait_for_migrations_on(&db, &config, Some(2), Some(1)).await; assert!( result.is_err(), @@ -1134,6 +1125,7 @@ mod tests { ..SQLiteConfig::default() }), postgres: None, + ..DatabaseConfig::default() }, pdf: codex_config::PdfConfig { cache_dir: pdf_cache_dir.to_string_lossy().to_string(), @@ -1258,7 +1250,10 @@ mod tests { #[tokio::test] async fn test_get_worker_count_from_config() { - let task_config = TaskConfig { worker_count: 8 }; + let task_config = TaskConfig { + worker_count: 8, + ..TaskConfig::default() + }; let worker_count = get_worker_count(Some(&task_config), None).await; assert_eq!(worker_count, 8); } @@ -1291,7 +1286,10 @@ mod tests { ); // Config should be used when provided (task.worker_count is now in config, not database) - let task_config = TaskConfig { worker_count: 5 }; + let task_config = TaskConfig { + worker_count: 5, + ..TaskConfig::default() + }; let worker_count = get_worker_count(Some(&task_config), Some(&settings_service)).await; assert_eq!(worker_count, 5); // Config value takes priority } @@ -1312,70 +1310,58 @@ mod tests { pragmas: None, ..SQLiteConfig::default() }), + ..DatabaseConfig::default() }, ..Config::default() } } + /// A config that leaves migrations to something else, with a short budget + /// so the waiting tests do not sit for the five-minute default. + fn skip_migrations(db_path: &std::path::Path, timeout: u64) -> Config { + let mut config = make_sqlite_config(db_path); + config.database.run_migrations = false; + config.database.migration_wait_timeout_secs = timeout; + config.database.migration_wait_interval_secs = 1; + config + } + #[tokio::test] - #[serial_test::serial(codex_migration_env)] async fn init_database_runs_migrations_by_default() { let temp_dir = TempDir::new().unwrap(); let db_path = temp_dir.path().join("test.db"); - unsafe { - std::env::remove_var("CODEX_SKIP_MIGRATIONS"); - } let config = make_sqlite_config(&db_path); let db = init_database(&config) .await - .expect("init_database should succeed when skip is unset"); + .expect("init_database should apply migrations by default"); - let complete = db.migrations_complete().await.unwrap(); - assert!(complete, "migrations should be complete"); + assert!(db.migrations_complete().await.unwrap()); } #[tokio::test] - #[serial_test::serial(codex_migration_env)] - async fn init_database_with_skip_succeeds_when_migrations_already_complete() { + async fn init_database_succeeds_when_migrations_already_ran_elsewhere() { let temp_dir = TempDir::new().unwrap(); let db_path = temp_dir.path().join("test.db"); let config = make_sqlite_config(&db_path); - - // Run migrations out of band first. let db = Database::new(&config.database).await.unwrap(); db.run_migrations().await.unwrap(); drop(db); - unsafe { - std::env::set_var("CODEX_SKIP_MIGRATIONS", "true"); - } - let result = init_database(&config).await; - unsafe { - std::env::remove_var("CODEX_SKIP_MIGRATIONS"); - } + let result = init_database(&skip_migrations(&db_path, 10)).await; assert!( result.is_ok(), - "init_database should succeed when skip is set and migrations are done: {:?}", - result + "should succeed when the schema is already current: {result:?}" ); } #[tokio::test] - #[serial_test::serial(codex_migration_env)] - async fn init_database_with_skip_waits_for_concurrent_migrations() { + async fn init_database_waits_for_migrations_running_elsewhere() { let temp_dir = TempDir::new().unwrap(); let db_path = temp_dir.path().join("test.db"); - - unsafe { - std::env::set_var("CODEX_SKIP_MIGRATIONS", "true"); - std::env::set_var("CODEX_MIGRATION_WAIT_TIMEOUT", "10"); - std::env::set_var("CODEX_MIGRATION_WAIT_INTERVAL", "1"); - } - - let config = make_sqlite_config(&db_path); + let config = skip_migrations(&db_path, 10); let config_clone = config.clone(); let migration_handle = tokio::spawn(async move { @@ -1387,70 +1373,22 @@ mod tests { let result = init_database(&config).await; migration_handle.await.unwrap(); - unsafe { - std::env::remove_var("CODEX_SKIP_MIGRATIONS"); - std::env::remove_var("CODEX_MIGRATION_WAIT_TIMEOUT"); - std::env::remove_var("CODEX_MIGRATION_WAIT_INTERVAL"); - } - let db = result.expect("init_database should succeed once migrations complete"); assert!(db.migrations_complete().await.unwrap()); } #[tokio::test] - #[serial_test::serial(codex_migration_env)] - async fn init_database_with_skip_accepts_one_as_truthy() { - let temp_dir = TempDir::new().unwrap(); - let db_path = temp_dir.path().join("test.db"); - - let config = make_sqlite_config(&db_path); - - let db = Database::new(&config.database).await.unwrap(); - db.run_migrations().await.unwrap(); - drop(db); - - unsafe { - std::env::set_var("CODEX_SKIP_MIGRATIONS", "1"); - } - let result = init_database(&config).await; - unsafe { - std::env::remove_var("CODEX_SKIP_MIGRATIONS"); - } - - assert!( - result.is_ok(), - "'1' should be treated as truthy: {:?}", - result - ); - } - - #[tokio::test] - #[serial_test::serial(codex_migration_env)] - async fn init_database_with_skip_times_out_when_migrations_never_run() { + async fn init_database_times_out_when_migrations_never_run() { let temp_dir = TempDir::new().unwrap(); let db_path = temp_dir.path().join("test.db"); - unsafe { - std::env::set_var("CODEX_SKIP_MIGRATIONS", "true"); - std::env::set_var("CODEX_MIGRATION_WAIT_TIMEOUT", "2"); - std::env::set_var("CODEX_MIGRATION_WAIT_INTERVAL", "1"); - } - - let config = make_sqlite_config(&db_path); - let result = init_database(&config).await; - - unsafe { - std::env::remove_var("CODEX_SKIP_MIGRATIONS"); - std::env::remove_var("CODEX_MIGRATION_WAIT_TIMEOUT"); - std::env::remove_var("CODEX_MIGRATION_WAIT_INTERVAL"); - } + let err = init_database(&skip_migrations(&db_path, 2)) + .await + .expect_err("should time out when migrations never complete"); - let err = result.expect_err("should time out when migrations never complete"); - let msg = err.to_string(); assert!( - msg.to_lowercase().contains("timeout"), - "error should mention timeout: {}", - msg + err.to_string().to_lowercase().contains("timeout"), + "error should mention timeout: {err}" ); } } diff --git a/crates/codex-config/src/de.rs b/crates/codex-config/src/de.rs index 0c1172aba..725b9d099 100644 --- a/crates/codex-config/src/de.rs +++ b/crates/codex-config/src/de.rs @@ -24,47 +24,97 @@ pub fn truthy_bool<'de, D>(deserializer: D) -> Result where D: Deserializer<'de>, { - struct TruthyBool; + deserializer.deserialize_any(TruthyBool) +} + +struct TruthyBool; + +impl<'de> Visitor<'de> for TruthyBool { + type Value = bool; + + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("a boolean: true/false, 1/0, yes/no or on/off") + } + + fn visit_bool(self, value: bool) -> Result { + Ok(value) + } + + fn visit_u64(self, value: u64) -> Result { + match value { + 0 => Ok(false), + 1 => Ok(true), + other => Err(E::custom(format!("expected 0 or 1, found {other}"))), + } + } + + fn visit_i64(self, value: i64) -> Result { + match value { + 0 => Ok(false), + 1 => Ok(true), + other => Err(E::custom(format!("expected 0 or 1, found {other}"))), + } + } + + fn visit_str(self, value: &str) -> Result { + match value.trim().to_ascii_lowercase().as_str() { + "true" | "1" | "yes" | "on" => Ok(true), + "false" | "0" | "no" | "off" => Ok(false), + other => Err(E::custom(format!( + "expected true/false, 1/0, yes/no or on/off, found `{other}`" + ))), + } + } +} - impl<'de> Visitor<'de> for TruthyBool { - type Value = bool; +/// An optional bool with the same lenient spellings. +/// +/// Needed because `Option` does not route through [`truthy_bool`]: +/// serde hands the inner value to the option's own deserializer, so +/// `CODEX_AUTH__COOKIE_SECURE=1` would otherwise fail with "expected a +/// boolean". +pub fn optional_truthy_bool<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + struct OptionalTruthyBool; + + impl<'de> Visitor<'de> for OptionalTruthyBool { + type Value = Option; fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.write_str("a boolean: true/false, 1/0, yes/no or on/off") + f.write_str("a boolean, or nothing") } - fn visit_bool(self, value: bool) -> Result { - Ok(value) + fn visit_none(self) -> Result { + Ok(None) } - fn visit_u64(self, value: u64) -> Result { - match value { - 0 => Ok(false), - 1 => Ok(true), - other => Err(E::custom(format!("expected 0 or 1, found {other}"))), - } + fn visit_unit(self) -> Result { + Ok(None) } - fn visit_i64(self, value: i64) -> Result { - match value { - 0 => Ok(false), - 1 => Ok(true), - other => Err(E::custom(format!("expected 0 or 1, found {other}"))), - } + fn visit_bool(self, value: bool) -> Result { + Ok(Some(value)) } - fn visit_str(self, value: &str) -> Result { - match value.trim().to_ascii_lowercase().as_str() { - "true" | "1" | "yes" | "on" => Ok(true), - "false" | "0" | "no" | "off" => Ok(false), - other => Err(E::custom(format!( - "expected true/false, 1/0, yes/no or on/off, found `{other}`" - ))), + fn visit_u64(self, value: u64) -> Result { + TruthyBool.visit_u64(value).map(Some) + } + + fn visit_str(self, value: &str) -> Result { + if value.is_empty() { + return Ok(None); } + TruthyBool.visit_str(value).map(Some) + } + + fn visit_some>(self, inner: D2) -> Result { + inner.deserialize_any(TruthyBool).map(Some) } } - deserializer.deserialize_any(TruthyBool) + deserializer.deserialize_option(OptionalTruthyBool) } /// A list of strings from a real list or a comma-separated string. diff --git a/crates/codex-config/src/env_audit.rs b/crates/codex-config/src/env_audit.rs index 0a6cfa9d5..65634bae2 100644 --- a/crates/codex-config/src/env_audit.rs +++ b/crates/codex-config/src/env_audit.rs @@ -31,26 +31,60 @@ use std::sync::OnceLock; /// by `codex-api`'s build script via `cargo:rustc-env` and read with `env!()`, /// so it never exists in a running process's environment. pub const NON_CONFIG_VARS: &[&str] = &[ - // Cookie `Secure` attribute override, applied per-response. - "CODEX_COOKIE_SECURE", - // Runs `serve` without in-process task workers. - "CODEX_DISABLE_WORKERS", - // Credential encryption key, read deep in codex-utils / codex-db. + // Credential encryption key. Read at its point of use deep in + // codex-utils / codex-db, which have no configuration in scope; bringing + // it into `Config` means threading config through those crates. "CODEX_ENCRYPTION_KEY", - // Bound on concurrent image decodes. - "CODEX_IMAGE_DECODE_CONCURRENCY", - // How often, and for how long, to poll while waiting on migrations. - "CODEX_MIGRATION_WAIT_INTERVAL", - "CODEX_MIGRATION_WAIT_TIMEOUT", - // Extra executables plugins are permitted to spawn. - "CODEX_PLUGIN_ALLOWED_COMMANDS", - // Leaves migrations to an external job and waits for them instead. - "CODEX_SKIP_MIGRATIONS", - // Per-invocation endpoints for `codex copy`; also available as CLI flags. + // Per-invocation endpoints for `codex copy`, also available as CLI flags. "CODEX_SOURCE_DATABASE_URL", "CODEX_TARGET_DATABASE_URL", ]; +/// Variables that were removed in favour of a real config key. +/// +/// These cannot be derived by re-spelling, either because the setting was +/// renamed outright or because its sense was inverted, so they need saying +/// explicitly. Getting one of them wrong is not cosmetic: a deployment that +/// keeps `CODEX_DISABLE_WORKERS=true` and is not told about it starts task +/// workers in a pod meant to serve web traffic only. +pub const REMOVED_VARS: &[(&str, &str, &str)] = &[ + ( + "CODEX_COOKIE_SECURE", + "CODEX_AUTH__COOKIE_SECURE", + "same meaning", + ), + ( + "CODEX_DISABLE_WORKERS", + "CODEX_TASK__RUN_IN_PROCESS", + "INVERTED: `DISABLE_WORKERS=true` becomes `RUN_IN_PROCESS=false`", + ), + ( + "CODEX_IMAGE_DECODE_CONCURRENCY", + "CODEX_IMAGES__DECODE_CONCURRENCY", + "same meaning", + ), + ( + "CODEX_MIGRATION_WAIT_INTERVAL", + "CODEX_DATABASE__MIGRATION_WAIT_INTERVAL_SECS", + "same meaning", + ), + ( + "CODEX_MIGRATION_WAIT_TIMEOUT", + "CODEX_DATABASE__MIGRATION_WAIT_TIMEOUT_SECS", + "same meaning", + ), + ( + "CODEX_PLUGIN_ALLOWED_COMMANDS", + "CODEX_PLUGINS__ALLOWED_COMMANDS", + "same meaning", + ), + ( + "CODEX_SKIP_MIGRATIONS", + "CODEX_DATABASE__RUN_MIGRATIONS", + "INVERTED: `SKIP_MIGRATIONS=true` becomes `RUN_MIGRATIONS=false`", + ), +]; + /// What the classifier concluded about one environment variable. /// /// Carries data rather than rendered text so the same findings can be printed @@ -74,6 +108,15 @@ pub enum Finding { /// The name this version does read. v1_name: String, }, + /// Removed in favour of a config key that cannot be derived by + /// re-spelling the old name. + Removed { + var: String, + /// The variable to set instead. + replacement: String, + /// How the meaning maps over, including any inversion. + note: String, + }, /// Not a recognized setting. Never fatal: a sibling tool may legitimately /// use the `CODEX_` prefix. Unknown { @@ -89,13 +132,17 @@ impl Finding { match self { Finding::WillRename { var, .. } | Finding::NotYetValid { var, .. } + | Finding::Removed { var, .. } | Finding::Unknown { var, .. } => var, } } /// Whether this finding means a setting is being ignored right now. pub fn is_ignored_now(&self) -> bool { - matches!(self, Finding::NotYetValid { .. } | Finding::Unknown { .. }) + matches!( + self, + Finding::NotYetValid { .. } | Finding::Removed { .. } | Finding::Unknown { .. } + ) } } @@ -154,6 +201,17 @@ pub fn classify(var: &str, registry: &KeyRegistry) -> Option { return None; } + // Checked before any spelling heuristic: these moved to a differently + // named key, sometimes with the sense flipped, so guessing would be worse + // than saying nothing. + if let Some((_, replacement, note)) = REMOVED_VARS.iter().find(|(old, _, _)| *old == var) { + return Some(Finding::Removed { + var: var.to_string(), + replacement: (*replacement).to_string(), + note: (*note).to_string(), + }); + } + // A `__` anywhere means the operator wrote a v2-style name. if rest.contains("__") { let path = rest @@ -491,6 +549,68 @@ mod tests { /// The compile-time build-script value must not be on the allowlist: it /// never appears in a running process's environment. + /// Three entries, not ten: the operator-tunable settings moved into + /// `Config`, leaving only a secret read too deep to reach and two + /// per-invocation arguments. + #[test] + fn the_allowlist_is_only_what_cannot_be_config() { + assert_eq!( + NON_CONFIG_VARS, + [ + "CODEX_ENCRYPTION_KEY", + "CODEX_SOURCE_DATABASE_URL", + "CODEX_TARGET_DATABASE_URL" + ] + ); + } + + /// Every removed variable must report its replacement. Silence here means + /// a deployment keeps a setting that no longer does anything. + #[test] + fn removed_variables_name_their_replacement() { + for (old, replacement, _) in REMOVED_VARS { + match classify_one(old) { + Some(Finding::Removed { + replacement: got, .. + }) => assert_eq!(&got, replacement, "wrong replacement for {old}"), + other => panic!("{old} should be Removed, got {other:?}"), + } + } + } + + /// The two inverted ones are the dangerous pair: keeping the old variable + /// and ignoring it silently flips behaviour. + #[test] + fn inverted_replacements_say_so() { + for var in ["CODEX_DISABLE_WORKERS", "CODEX_SKIP_MIGRATIONS"] { + match classify_one(var) { + Some(Finding::Removed { note, .. }) => assert!( + note.contains("INVERTED"), + "{var} flips sense and must say so, got: {note}" + ), + other => panic!("{var} should be Removed, got {other:?}"), + } + } + } + + /// Their new names must resolve as ordinary settings. + #[test] + fn the_replacements_are_real_settings() { + for (_, replacement, _) in REMOVED_VARS { + let path = replacement + .strip_prefix(ENV_PREFIX) + .unwrap() + .split("__") + .map(str::to_lowercase) + .collect::>() + .join("."); + assert!( + registry().contains(&path), + "{replacement} maps to `{path}`, which is not a config key" + ); + } + } + #[test] fn bin_version_is_not_allowlisted() { assert!(!NON_CONFIG_VARS.contains(&"CODEX_BIN_VERSION")); diff --git a/crates/codex-config/src/keys.rs b/crates/codex-config/src/keys.rs index 10eec09b8..04a42f35a 100644 --- a/crates/codex-config/src/keys.rs +++ b/crates/codex-config/src/keys.rs @@ -133,6 +133,11 @@ fn walk(value: &Value, path: &str, out: &mut KeyRegistry) { /// Two tests guard this against drift: the serialized probe must contain no /// `null` (which would mean an `Option` was left unpopulated) and no empty /// object (which would mean a map was left empty). +/// +/// Those two do not cover an `Option` carrying +/// `skip_serializing_if = "Option::is_none"`, which disappears entirely rather +/// than serializing as `null`. Such fields must be listed in +/// `optional_leaves_are_present` as well. fn key_probe() -> Config { let mut config = Config::default(); @@ -143,6 +148,7 @@ fn key_probe() -> Config { }); config.application.base_url = Some(String::new()); + config.auth.cookie_secure = Some(false); config.logging.file = Some(String::new()); config.email.verification_url_base = Some(String::new()); config.pdf.pdfium_library_path = Some(String::new()); @@ -255,6 +261,9 @@ mod tests { "pdf.pdfium_library_path", "observability.otlp.proxy_endpoint", "auth.oidc.redirect_uri_base", + // Carries `skip_serializing_if`, so it vanishes from the probe + // rather than showing up as `null`; the drift tests cannot see it. + "auth.cookie_secure", ] { assert!(registry.contains(path), "missing `{path}`"); } diff --git a/crates/codex-config/src/lib.rs b/crates/codex-config/src/lib.rs index 4703c1cca..45c29061d 100644 --- a/crates/codex-config/src/lib.rs +++ b/crates/codex-config/src/lib.rs @@ -13,11 +13,11 @@ mod types; #[allow(unused_imports)] pub use types::{ ApiConfig, ApplicationConfig, AuthConfig, Config, ConfigError, DatabaseConfig, DatabaseType, - EmailConfig, FilesConfig, KomgaApiConfig, KoreaderApiConfig, LoggingConfig, + EmailConfig, FilesConfig, ImagesConfig, KomgaApiConfig, KoreaderApiConfig, LoggingConfig, ObservabilityBrowserConfig, ObservabilityConfig, ObservabilityMetricsConfig, ObservabilityTracesConfig, OidcConfig, OidcDefaultRole, OidcProviderConfig, OtlpConfig, - OtlpProtocol, PdfConfig, PdfHandleCacheConfig, PostgresConfig, RateLimitConfig, SQLiteConfig, - ScannerConfig, SchedulerConfig, TaskConfig, + OtlpProtocol, PdfConfig, PdfHandleCacheConfig, PluginsConfig, PostgresConfig, RateLimitConfig, + SQLiteConfig, ScannerConfig, SchedulerConfig, TaskConfig, }; pub use env_audit::{ diff --git a/crates/codex-config/src/loader.rs b/crates/codex-config/src/loader.rs index 3fd98fb60..e8c5c2d4e 100644 --- a/crates/codex-config/src/loader.rs +++ b/crates/codex-config/src/loader.rs @@ -708,6 +708,79 @@ auth: }); } + // ---- settings that used to be ad-hoc environment reads ---- + + /// Each of these was `std::env::var` at its point of use. They must now be + /// settable from the file and from the environment like anything else. + #[test] + fn the_relocated_settings_are_real_config_keys() { + Jail::expect_with(|jail| { + jail.create_file( + "codex.yaml", + r#" +auth: + cookie_secure: true +task: + run_in_process: false +images: + decode_concurrency: 9 +plugins: + allowed_commands: deno,bun +database: + run_migrations: false + migration_wait_timeout_secs: 42 + migration_wait_interval_secs: 7 +"#, + )?; + + let config = Config::load(jail.directory().join("codex.yaml")).unwrap(); + + assert_eq!(config.auth.cookie_secure, Some(true)); + assert!(!config.task.run_in_process); + assert_eq!(config.images.decode_concurrency, 9); + assert_eq!(config.plugins.allowed_commands, vec!["deno", "bun"]); + assert!(!config.database.run_migrations); + assert_eq!(config.database.migration_wait_timeout_secs, 42); + assert_eq!(config.database.migration_wait_interval_secs, 7); + Ok(()) + }); + } + + #[test] + fn the_relocated_settings_are_settable_from_the_environment() { + Jail::expect_with(|jail| { + jail.set_env("CODEX_AUTH__COOKIE_SECURE", "1"); + jail.set_env("CODEX_TASK__RUN_IN_PROCESS", "false"); + jail.set_env("CODEX_IMAGES__DECODE_CONCURRENCY", "12"); + jail.set_env("CODEX_PLUGINS__ALLOWED_COMMANDS", "deno, bun"); + jail.set_env("CODEX_DATABASE__RUN_MIGRATIONS", "no"); + jail.set_env("CODEX_DATABASE__MIGRATION_WAIT_TIMEOUT_SECS", "60"); + + let config = Config::load(jail.directory().join("absent.yaml")).unwrap(); + + assert_eq!(config.auth.cookie_secure, Some(true)); + assert!(!config.task.run_in_process); + assert_eq!(config.images.decode_concurrency, 12); + assert_eq!(config.plugins.allowed_commands, vec!["deno", "bun"]); + assert!(!config.database.run_migrations); + assert_eq!(config.database.migration_wait_timeout_secs, 60); + Ok(()) + }); + } + + /// The inverted pair must default to today's behaviour: workers run and + /// migrations are applied unless something says otherwise. + #[test] + fn the_inverted_settings_default_to_the_previous_behaviour() { + let config = Config::default(); + assert!(config.task.run_in_process, "workers ran by default before"); + assert!( + config.database.run_migrations, + "migrations were applied by default before" + ); + assert!(!config.auth.cookie_secure(), "Secure was off by default"); + } + #[test] fn overlay_path_gets_a_local_infix() { assert_eq!( diff --git a/crates/codex-config/src/testdata/default_config.json b/crates/codex-config/src/testdata/default_config.json index 6a5388342..3f4b6ec0f 100644 --- a/crates/codex-config/src/testdata/default_config.json +++ b/crates/codex-config/src/testdata/default_config.json @@ -2,11 +2,14 @@ "data_dir": "data", "database": { "db_type": "sqlite", + "run_migrations": true, + "migration_wait_timeout_secs": 300, + "migration_wait_interval_secs": 2, "sqlite": { "path": "data/codex.db", "pragmas": { - "journal_mode": "WAL", - "foreign_keys": "ON" + "foreign_keys": "ON", + "journal_mode": "WAL" }, "max_connections": 64, "min_connections": 2, @@ -64,11 +67,18 @@ "verification_token_expiry_hours": 24 }, "task": { + "run_in_process": true, "worker_count": 2 }, "scanner": { "max_concurrent_scans": 2 }, + "images": { + "decode_concurrency": 3 + }, + "plugins": { + "allowed_commands": [] + }, "scheduler": { "timezone": "UTC" }, diff --git a/crates/codex-config/src/types.rs b/crates/codex-config/src/types.rs index 8c48b8147..049ffd4c8 100644 --- a/crates/codex-config/src/types.rs +++ b/crates/codex-config/src/types.rs @@ -5,6 +5,14 @@ use std::collections::HashMap; #[derive(Debug, Serialize, Deserialize, Clone)] #[serde(default)] pub struct TaskConfig { + /// Run task workers inside this process. + /// + /// Set false for a web-only replica in a deployment that runs workers in + /// their own pods. Note that SSE task progress only reaches the browser + /// when workers share the process, unless a progress bridge is configured. + #[serde(deserialize_with = "crate::de::truthy_bool")] + pub run_in_process: bool, + /// Number of parallel task workers to process tasks from the queue /// This is a startup-time setting - changes require a restart pub worker_count: u32, @@ -274,7 +282,10 @@ impl Default for KomgaApiConfig { impl Default for TaskConfig { fn default() -> Self { - Self { worker_count: 2 } + Self { + worker_count: 2, + run_in_process: true, + } } } @@ -303,6 +314,10 @@ pub struct Config { #[serde(default)] pub scanner: ScannerConfig, #[serde(default)] + pub images: ImagesConfig, + #[serde(default)] + pub plugins: PluginsConfig, + #[serde(default)] pub scheduler: SchedulerConfig, #[serde(default)] pub files: FilesConfig, @@ -466,6 +481,7 @@ impl Default for Config { db_type, postgres: postgres_config, sqlite: sqlite_config, + ..DatabaseConfig::default() }, application: ApplicationConfig::default(), logging: LoggingConfig::default(), @@ -474,6 +490,8 @@ impl Default for Config { email: EmailConfig::default(), task: TaskConfig::default(), scanner: ScannerConfig::default(), + images: ImagesConfig::default(), + plugins: PluginsConfig::default(), scheduler: SchedulerConfig::default(), files: FilesConfig::default(), pdf: PdfConfig::default(), @@ -490,6 +508,18 @@ impl Default for Config { #[serde(default)] pub struct AuthConfig { pub jwt_secret: String, + + /// Send the `Secure` attribute on auth cookies. + /// + /// Left unset it is off, which is what plain-HTTP development needs. Any + /// deployment terminating TLS should set it to true, or the session cookie + /// will also be sent over a plaintext downgrade. + #[serde( + default, + skip_serializing_if = "Option::is_none", + deserialize_with = "crate::de::optional_truthy_bool" + )] + pub cookie_secure: Option, pub jwt_expiry_hours: u32, #[serde(deserialize_with = "crate::de::truthy_bool")] pub refresh_token_enabled: bool, @@ -504,10 +534,18 @@ pub struct AuthConfig { pub oidc: OidcConfig, } +impl AuthConfig { + /// Whether auth cookies carry `Secure`. Unset means off. + pub fn cookie_secure(&self) -> bool { + self.cookie_secure.unwrap_or(false) + } +} + impl Default for AuthConfig { fn default() -> Self { Self { jwt_secret: "INSECURE_DEFAULT_SECRET_CHANGE_IN_PRODUCTION".to_string(), + cookie_secure: None, jwt_expiry_hours: 24, refresh_token_enabled: true, refresh_token_expiry_days: 30, @@ -552,6 +590,20 @@ impl Default for ApiConfig { pub struct DatabaseConfig { pub db_type: DatabaseType, + /// Apply pending migrations at startup. + /// + /// Set false when a separate Job or init container owns migrations; this + /// process then waits for the schema to be current instead of applying it. + #[serde(deserialize_with = "crate::de::truthy_bool")] + pub run_migrations: bool, + + /// How long to wait for the database to accept connections, and for its + /// schema to be current, before giving up. + pub migration_wait_timeout_secs: u64, + + /// How often to re-check while waiting. + pub migration_wait_interval_secs: u64, + // Postgres Specific #[serde(skip_serializing_if = "Option::is_none")] pub postgres: Option, @@ -634,6 +686,9 @@ impl Default for DatabaseConfig { Self { db_type, + run_migrations: true, + migration_wait_timeout_secs: 300, + migration_wait_interval_secs: 2, postgres: postgres_config, sqlite: sqlite_config, } @@ -896,6 +951,37 @@ impl LogLevel { } } +/// Image decode/resize/render limits. +#[derive(Debug, Serialize, Deserialize, Clone)] +#[serde(default)] +pub struct ImagesConfig { + /// Maximum concurrent decode/resize/render operations. + /// + /// Each permit corresponds to one in-flight uncompressed bitmap, so peak + /// image memory is roughly this times the per-decode footprint. Kept small + /// so a reader prefetching many pages cannot hold many full-resolution + /// bitmaps at once. + pub decode_concurrency: usize, +} + +impl Default for ImagesConfig { + fn default() -> Self { + Self { + decode_concurrency: 3, + } + } +} + +/// Plugin runtime settings. +#[derive(Debug, Serialize, Deserialize, Clone, Default)] +#[serde(default)] +pub struct PluginsConfig { + /// Executables plugins may spawn, in addition to the built-in runtimes + /// (`node`, `npx`, `python`, `python3`, `uv`, `uvx`). + #[serde(deserialize_with = "crate::de::string_list")] + pub allowed_commands: Vec, +} + #[derive(Debug, Serialize, Deserialize, Clone)] #[serde(default)] pub struct ScannerConfig { @@ -1485,6 +1571,9 @@ verification_url_base: https://codex.example.com fn test_database_config_postgres() { let config = DatabaseConfig { db_type: DatabaseType::Postgres, + run_migrations: true, + migration_wait_timeout_secs: 300, + migration_wait_interval_secs: 2, postgres: Some(PostgresConfig { host: "localhost".to_string(), port: 5432, @@ -1505,6 +1594,9 @@ verification_url_base: https://codex.example.com fn test_database_config_sqlite() { let config = DatabaseConfig { db_type: DatabaseType::SQLite, + run_migrations: true, + migration_wait_timeout_secs: 300, + migration_wait_interval_secs: 2, postgres: None, sqlite: Some(SQLiteConfig { path: "/var/lib/codex.db".to_string(), @@ -1522,6 +1614,9 @@ verification_url_base: https://codex.example.com fn test_operation_deadline_seconds_sqlite() { let config = DatabaseConfig { db_type: DatabaseType::SQLite, + run_migrations: true, + migration_wait_timeout_secs: 300, + migration_wait_interval_secs: 2, postgres: None, sqlite: Some(SQLiteConfig { path: "./test.db".to_string(), @@ -1538,6 +1633,9 @@ verification_url_base: https://codex.example.com fn test_operation_deadline_seconds_postgres() { let config = DatabaseConfig { db_type: DatabaseType::Postgres, + run_migrations: true, + migration_wait_timeout_secs: 300, + migration_wait_interval_secs: 2, postgres: Some(PostgresConfig { host: "localhost".to_string(), port: 5432, @@ -1565,6 +1663,9 @@ verification_url_base: https://codex.example.com // SQLite: explicit values flow through the accessors. let sqlite = DatabaseConfig { db_type: DatabaseType::SQLite, + run_migrations: true, + migration_wait_timeout_secs: 300, + migration_wait_interval_secs: 2, postgres: None, sqlite: Some(SQLiteConfig { batch_fan_out: 6, @@ -1580,6 +1681,9 @@ verification_url_base: https://codex.example.com // batch_fan_out is clamped to at least 1 (a 0 would deadlock a request). let zero = DatabaseConfig { db_type: DatabaseType::SQLite, + run_migrations: true, + migration_wait_timeout_secs: 300, + migration_wait_interval_secs: 2, postgres: None, sqlite: Some(SQLiteConfig { batch_fan_out: 0, @@ -1591,6 +1695,9 @@ verification_url_base: https://codex.example.com // Postgres: explicit values flow through the accessors. let postgres = DatabaseConfig { db_type: DatabaseType::Postgres, + run_migrations: true, + migration_wait_timeout_secs: 300, + migration_wait_interval_secs: 2, postgres: Some(PostgresConfig { batch_fan_out: 12, background_max_connections: 20, @@ -1628,6 +1735,9 @@ verification_url_base: https://codex.example.com data_dir: "data".to_string(), database: DatabaseConfig { db_type: DatabaseType::SQLite, + run_migrations: true, + migration_wait_timeout_secs: 300, + migration_wait_interval_secs: 2, postgres: None, sqlite: Some(SQLiteConfig { path: "./codex.db".to_string(), @@ -1646,6 +1756,8 @@ verification_url_base: https://codex.example.com email: EmailConfig::default(), task: TaskConfig::default(), scanner: ScannerConfig::default(), + images: ImagesConfig::default(), + plugins: PluginsConfig::default(), scheduler: SchedulerConfig::default(), files: FilesConfig::default(), pdf: PdfConfig::default(), @@ -1711,6 +1823,7 @@ verification_url_base: https://codex.example.com argon2_time_cost: 3, argon2_parallelism: 2, oidc: OidcConfig::default(), + ..AuthConfig::default() }; let yaml = serde_yaml::to_string(&config).unwrap(); @@ -1752,7 +1865,10 @@ verification_url_base: https://codex.example.com #[test] fn test_task_config_serialization() { - let config = TaskConfig { worker_count: 8 }; + let config = TaskConfig { + worker_count: 8, + ..TaskConfig::default() + }; let yaml = serde_yaml::to_string(&config).unwrap(); assert!(yaml.contains("worker_count")); diff --git a/crates/codex-db/src/connection.rs b/crates/codex-db/src/connection.rs index e1d4ce3b4..6b9280169 100644 --- a/crates/codex-db/src/connection.rs +++ b/crates/codex-db/src/connection.rs @@ -531,6 +531,7 @@ mod tests { pragmas: None, ..SQLiteConfig::default() }), + ..DatabaseConfig::default() }; let db = Database::new(&config).await.unwrap(); @@ -561,6 +562,7 @@ mod tests { max_connections: 16, ..SQLiteConfig::default() }), + ..DatabaseConfig::default() }; // Primary pool owns schema setup. @@ -615,6 +617,7 @@ mod tests { ..codex_config::PostgresConfig::default() }), sqlite: None, + ..DatabaseConfig::default() }; let capped = Database::with_pool_max(&config, 1); @@ -641,6 +644,7 @@ mod tests { min_connections: 8, ..SQLiteConfig::default() }), + ..DatabaseConfig::default() }; let probe = Database::new_probe(&config).await.unwrap(); @@ -674,6 +678,7 @@ mod tests { ..codex_config::PostgresConfig::default() }), sqlite: None, + ..DatabaseConfig::default() }; let db = Database::new(&config).await.unwrap(); diff --git a/crates/codex-db/src/test_helpers.rs b/crates/codex-db/src/test_helpers.rs index e718cbcf6..02437425a 100644 --- a/crates/codex-db/src/test_helpers.rs +++ b/crates/codex-db/src/test_helpers.rs @@ -30,6 +30,7 @@ pub async fn create_test_db() -> (Database, TempDir) { pragmas: Some(pragmas), ..SQLiteConfig::default() }), + ..DatabaseConfig::default() }; let db = Database::new(&config).await.unwrap(); diff --git a/crates/codex-migrate/src/conn.rs b/crates/codex-migrate/src/conn.rs index d383c0649..27436deeb 100644 --- a/crates/codex-migrate/src/conn.rs +++ b/crates/codex-migrate/src/conn.rs @@ -30,6 +30,7 @@ pub fn database_config_from_url(raw: &str) -> Result { path: path.to_string(), ..SQLiteConfig::default() }), + ..DatabaseConfig::default() }); } @@ -55,6 +56,7 @@ pub fn database_config_from_url(raw: &str) -> Result { ..PostgresConfig::default() }), sqlite: None, + ..DatabaseConfig::default() }); } diff --git a/crates/codex-services/src/plugin/process.rs b/crates/codex-services/src/plugin/process.rs index b599bb122..602e58046 100644 --- a/crates/codex-services/src/plugin/process.rs +++ b/crates/codex-services/src/plugin/process.rs @@ -30,9 +30,6 @@ use tracing::{debug, error, info, warn}; // Command Allowlist // ============================================================================= -/// Environment variable for customizing allowed plugin commands -pub const ALLOWED_COMMANDS_ENV: &str = "CODEX_PLUGIN_ALLOWED_COMMANDS"; - /// Default allowed commands for plugin execution /// /// These are common runtimes used by plugins: @@ -161,26 +158,46 @@ pub fn filter_blocked_env_vars(env: &HashMap) -> HashMap &'static Vec { - COMMAND_ALLOWLIST.get_or_init(|| { - let mut allowlist: Vec = DEFAULT_ALLOWED_COMMANDS - .iter() - .map(|s| s.to_string()) - .collect(); - - // Add custom commands from environment variable - if let Ok(custom) = std::env::var(ALLOWED_COMMANDS_ENV) { - for cmd in custom.split(',') { - let cmd = cmd.trim(); - if !cmd.is_empty() && !allowlist.contains(&cmd.to_string()) { - allowlist.push(cmd.to_string()); - } - } +/// Install the configured extras on top of the built-in runtimes. +/// +/// Call once during startup, before anything can spawn a plugin. The allowlist +/// is a process-wide `OnceLock` because [`is_command_allowed`] is reached from +/// request handlers that have no configuration in scope; passing it down every +/// call path would mean threading config through the whole plugin surface for +/// a value that never changes after boot. +/// +/// A later call is ignored, so a first use that beats initialization cannot be +/// silently overridden by a different allowlist afterwards. +pub fn init_command_allowlist(extra: &[String]) { + if COMMAND_ALLOWLIST + .set(build_command_allowlist(extra)) + .is_err() + { + warn!( + "Plugin command allowlist was already initialized; \ + configured `plugins.allowed_commands` entries were not applied" + ); + } +} + +fn build_command_allowlist(extra: &[String]) -> Vec { + let mut allowlist: Vec = DEFAULT_ALLOWED_COMMANDS + .iter() + .map(|s| s.to_string()) + .collect(); + for command in extra { + let command = command.trim(); + if !command.is_empty() && !allowlist.iter().any(|a| a == command) { + allowlist.push(command.to_string()); } + } + allowlist +} - allowlist - }) +/// The allowlist, falling back to the built-in runtimes when startup never +/// installed one (tests, and any tool that does not load configuration). +fn get_command_allowlist() -> &'static Vec { + COMMAND_ALLOWLIST.get_or_init(|| build_command_allowlist(&[])) } /// Check if a command is in the allowlist diff --git a/crates/codex-services/src/refresh_token.rs b/crates/codex-services/src/refresh_token.rs index 4e17e8f5e..c18a853b8 100644 --- a/crates/codex-services/src/refresh_token.rs +++ b/crates/codex-services/src/refresh_token.rs @@ -233,6 +233,7 @@ mod tests { pragmas: Some(pragmas), ..SQLiteConfig::default() }), + ..DatabaseConfig::default() }; let db = Database::new(&config).await.unwrap(); diff --git a/crates/codex-tasks/src/handlers/cleanup_refresh_tokens.rs b/crates/codex-tasks/src/handlers/cleanup_refresh_tokens.rs index 35de04650..04b99af50 100644 --- a/crates/codex-tasks/src/handlers/cleanup_refresh_tokens.rs +++ b/crates/codex-tasks/src/handlers/cleanup_refresh_tokens.rs @@ -81,6 +81,7 @@ mod tests { pragmas: Some(pragmas), ..SQLiteConfig::default() }), + ..DatabaseConfig::default() }; let db = Database::new(&config).await.unwrap(); diff --git a/src/commands/serve.rs b/src/commands/serve.rs index 9cd13f630..825ff5195 100644 --- a/src/commands/serve.rs +++ b/src/commands/serve.rs @@ -98,10 +98,8 @@ pub async fn serve_command(config_path: PathBuf) -> anyhow::Result<()> { info!("For SSE to work with SQLite, workers must run in the same process"); } - // Check if workers should be disabled (useful for web-only pods in k8s) - let disable_workers = std::env::var("CODEX_DISABLE_WORKERS") - .map(|v| v.eq_ignore_ascii_case("true") || v == "1") - .unwrap_or(false); + // Web-only replicas run workers in their own pods. + let disable_workers = !config.task.run_in_process; // Background connection pool isolation. // @@ -252,11 +250,7 @@ pub async fn serve_command(config_path: PathBuf) -> anyhow::Result<()> { // bitmaps in memory at once. Each permit maps to one in-flight uncompressed // bitmap, so peak image memory ≈ permits × per-decode footprint. Small by // default; env-tunable for larger boxes. - let image_decode_concurrency = std::env::var("CODEX_IMAGE_DECODE_CONCURRENCY") - .ok() - .and_then(|v| v.parse::().ok()) - .filter(|n| *n > 0) - .unwrap_or(3); + let image_decode_concurrency = config.images.decode_concurrency.max(1); codex_api::image_limit::init_image_decode_limiter(image_decode_concurrency); info!( "Image decode limiter initialized (max concurrent decode/resize/render: {})", @@ -422,6 +416,11 @@ pub async fn serve_command(config_path: PathBuf) -> anyhow::Result<()> { let scheduler_handle: codex_services::scheduler_handle::SharedSchedulerReconciler = Arc::new( codex_scheduler::LockedSchedulerReconciler::new(scheduler.clone()), ); + // Install the configured plugin command allowlist before anything can + // spawn a plugin; it is a process-wide value read from handlers that have + // no config in scope. + codex_services::plugin::process::init_command_allowlist(&config.plugins.allowed_commands); + let plugin_manager = Arc::new( codex_services::plugin::PluginManager::with_defaults(Arc::new( db.sea_orm_connection().clone(), @@ -459,19 +458,12 @@ pub async fn serve_command(config_path: PathBuf) -> anyhow::Result<()> { let mut worker_count = 0u32; if disable_workers { - info!("Workers disabled via CODEX_DISABLE_WORKERS environment variable"); + info!("Workers disabled (task.run_in_process is false)"); } else { // Get worker count from config (which includes env override) or settings fallback worker_count = get_worker_count(Some(&config.task), Some(&settings_service)).await; - if let Ok(env_count) = std::env::var("CODEX_TASK_WORKER_COUNT") { - info!( - "Worker count from environment variable CODEX_TASK_WORKER_COUNT: {}", - env_count - ); - } else { - info!("Worker count from settings: {}", worker_count); - } + info!("Worker count: {}", worker_count); // Reconcile orphaned series exports from prior crash/restart if let Err(e) = codex_tasks::handlers::cleanup_series_exports::reconcile_on_startup( diff --git a/src/commands/worker.rs b/src/commands/worker.rs index d89dac836..9ce28a209 100644 --- a/src/commands/worker.rs +++ b/src/commands/worker.rs @@ -52,14 +52,7 @@ pub async fn worker_command(config_path: PathBuf) -> anyhow::Result<()> { // Get worker count from config (which includes env override) or settings fallback let worker_count = get_worker_count(Some(&config.task), Some(&settings_service)).await; - if let Ok(env_count) = std::env::var("CODEX_TASK_WORKER_COUNT") { - info!( - "Worker count from environment variable CODEX_TASK_WORKER_COUNT: {}", - env_count - ); - } else { - info!("Worker count from config: {}", worker_count); - } + info!("Worker count: {}", worker_count); info!("Starting {} task queue worker(s)...", worker_count); @@ -172,6 +165,11 @@ pub async fn worker_command(config_path: PathBuf) -> anyhow::Result<()> { let plugin_file_storage = Arc::new(codex_services::PluginFileStorage::new( &config.files.plugins_dir, )); + // Install the configured plugin command allowlist before anything can + // spawn a plugin; it is a process-wide value read from handlers that have + // no config in scope. + codex_services::plugin::process::init_command_allowlist(&config.plugins.allowed_commands); + let plugin_manager = Arc::new( codex_services::plugin::PluginManager::with_defaults(Arc::new( db.sea_orm_connection().clone(), diff --git a/tests/api/pool_contention.rs b/tests/api/pool_contention.rs index 90e04d23a..27a63c9d5 100644 --- a/tests/api/pool_contention.rs +++ b/tests/api/pool_contention.rs @@ -60,6 +60,7 @@ async fn small_pool_sqlite() -> (sea_orm::DatabaseConnection, TempDir) { acquire_timeout_seconds: 5, ..SQLiteConfig::default() }), + ..DatabaseConfig::default() }; let database = Database::new(&config).await.unwrap(); diff --git a/tests/common/db.rs b/tests/common/db.rs index e26fd2fdf..0ee92ed91 100644 --- a/tests/common/db.rs +++ b/tests/common/db.rs @@ -21,6 +21,7 @@ pub async fn setup_test_db() -> (sea_orm::DatabaseConnection, TempDir) { pragmas: Some(pragmas), ..SQLiteConfig::default() }), + ..DatabaseConfig::default() }; let db = Database::new(&config).await.unwrap(); @@ -43,6 +44,7 @@ pub async fn setup_test_db_wrapper() -> (Database, TempDir) { pragmas: None, ..SQLiteConfig::default() }), + ..DatabaseConfig::default() }; let db = Database::new(&config).await.unwrap(); @@ -77,6 +79,7 @@ pub async fn setup_test_db_postgres() -> Option { database_name: extract_database(&postgres_url), ..PostgresConfig::default() }), + ..DatabaseConfig::default() }; // Try to create database connection diff --git a/tests/db/migrations.rs b/tests/db/migrations.rs index f9c72439f..e60f0718e 100644 --- a/tests/db/migrations.rs +++ b/tests/db/migrations.rs @@ -38,6 +38,7 @@ async fn test_migrations_complete_on_fresh_database() { pragmas: None, ..SQLiteConfig::default() }), + ..DatabaseConfig::default() }; let db = Database::new(&config).await.unwrap(); @@ -131,6 +132,7 @@ async fn setup_db_before_migration_056() -> (Database, TempDir) { pragmas: None, ..SQLiteConfig::default() }), + ..DatabaseConfig::default() }; let db = Database::new(&config).await.unwrap(); @@ -307,6 +309,7 @@ async fn test_migration_056_fresh_run_postgres() { ..PostgresConfig::default() }), sqlite: None, + ..DatabaseConfig::default() }; let db = match Database::new(&config).await { @@ -385,6 +388,7 @@ async fn setup_db_before_migration_067() -> (Database, TempDir) { pragmas: None, ..SQLiteConfig::default() }), + ..DatabaseConfig::default() }; let db = Database::new(&config).await.unwrap(); @@ -556,6 +560,7 @@ async fn setup_db_before_migration_069() -> (Database, TempDir) { pragmas: None, ..SQLiteConfig::default() }), + ..DatabaseConfig::default() }; let db = Database::new(&config).await.unwrap(); diff --git a/tests/db/postgres.rs b/tests/db/postgres.rs index 574e332f8..f6337ef84 100644 --- a/tests/db/postgres.rs +++ b/tests/db/postgres.rs @@ -37,6 +37,7 @@ async fn create_test_postgres_db() -> Database { ..PostgresConfig::default() }), sqlite: None, + ..DatabaseConfig::default() }; let db = Database::new(&config).await.unwrap(); diff --git a/tests/db/repositories.rs b/tests/db/repositories.rs index 14e381a23..b31fd1c26 100644 --- a/tests/db/repositories.rs +++ b/tests/db/repositories.rs @@ -949,6 +949,7 @@ async fn test_database_reconnect() { pragmas: None, ..SQLiteConfig::default() }), + ..DatabaseConfig::default() }; let db2 = Database::new(&config).await.unwrap(); From f5a634744d51a599dbb96f6a59e01078cf542bc2 Mon Sep 17 00:00:00 2001 From: Sylvain Cau Date: Sat, 15 Aug 2026 17:14:48 -0700 Subject: [PATCH 05/11] feat(config)!: refuse to start when the environment names a setting that is no longer read The environment audit shipped as advice, on the reasoning that the old names were still correct and 2.0 was not out. Both halves have now flipped: the nested spelling is what the loader reads, and a variable in the old form does nothing. Leaving that as a log line means a deployment carries on with default rate limits, or the wrong port, and the only evidence is one line in a startup log nobody reads. So `Config::load` now fails on it. Every offending variable appears in one message with its replacement, because an operator with a dozen of them should make one pass rather than discover them one restart at a time. Two severities, not one. A name that resolves to a real setting is fatal: the operator asked for something specific and is not getting it. A name that resolves to nothing stays a warning, because another tool may legitimately use the `CODEX_` prefix and guessing wrong should not take a deployment down. `Config::resolve` is the same resolution without the check, so `codex config check` can list every problem instead of stopping at the first. Its exit code now matches what the server would do: non-zero when a fatal finding is present, and `--strict` additionally fails on warnings. The `NotYetValid` finding is gone, since a nested name is now simply read, and the startup advisory line goes with it: enforcement says the same thing louder and at the right time. --- Cargo.lock | 1 + crates/codex-cli-common/src/lib.rs | 117 ++------------------ crates/codex-config/Cargo.toml | 1 + crates/codex-config/src/env_audit.rs | 158 ++++++++++++++++++--------- crates/codex-config/src/lib.rs | 4 +- crates/codex-config/src/loader.rs | 94 +++++++++++++++- crates/codex-config/src/types.rs | 6 + src/commands/config.rs | 129 +++++++++++----------- 8 files changed, 277 insertions(+), 233 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 76c342790..d22b32b99 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -961,6 +961,7 @@ dependencies = [ "serde_yaml", "serial_test", "tempfile", + "tracing", ] [[package]] diff --git a/crates/codex-cli-common/src/lib.rs b/crates/codex-cli-common/src/lib.rs index f001a627c..f6ce3280f 100644 --- a/crates/codex-cli-common/src/lib.rs +++ b/crates/codex-cli-common/src/lib.rs @@ -99,64 +99,21 @@ pub fn load_config(config_path: PathBuf) -> anyhow::Result<(Config, bool)> { false }; - let config = resolve_config(&config_path)?; - - warn_about_renamed_env_vars(); + let config = Config::load(&config_path)?; Ok((config, config_created)) } -/// Resolve configuration without touching the filesystem. +/// Resolve configuration for inspection: no file is created, and an +/// environment naming settings in the pre-2.0 form is reported by the caller +/// rather than rejected here. /// -/// A missing file yields the defaults rather than being created, so callers -/// that only want to *inspect* the configuration leave no trace. `config -/// check` relies on this: it is meant to run as a Kubernetes initContainer -/// against a read-only mount, and a validation step that writes a config file -/// as a side effect would be its own bug. +/// `config check` relies on both. It runs as a Kubernetes initContainer +/// against a read-only mount, so writing a config file as a side effect would +/// be its own bug, and it needs to list every problem rather than stop at the +/// first. pub fn resolve_config(config_path: &Path) -> anyhow::Result { - Config::load(config_path) -} - -/// Emit a single line when environment variables will need renaming in the -/// next major version, or are being ignored right now. -/// -/// Deliberately one line rather than one per variable. In this version the -/// flat names are still correct, so a warning per variable would be recurring -/// noise about something the operator cannot act on yet. `codex config check` -/// is where the detail lives. -fn warn_about_renamed_env_vars() { - if let Some(notice) = env_var_notice(&codex_config::audit_env()) { - warn!("{notice}"); - } -} - -/// The single advisory line, or `None` when there is nothing to say. -/// -/// Returning at most one string is the point: the count goes in the log and -/// the detail lives in `codex config check`. Naming each variable here would -/// put a growing block in every process's startup output on every boot. -fn env_var_notice(findings: &[codex_config::Finding]) -> Option { - if findings.is_empty() { - return None; - } - - let ignored = findings.iter().filter(|f| f.is_ignored_now()).count(); - let renamed = findings.len() - ignored; - - Some(match (renamed, ignored) { - (0, _) => format!( - "{ignored} environment variable(s) are not being read. \ - Run `codex config check` for details." - ), - (_, 0) => format!( - "{renamed} environment variable(s) will be renamed in Codex 2.0. \ - Run `codex config check` for the list." - ), - _ => format!( - "{renamed} environment variable(s) will be renamed in Codex 2.0 and \ - {ignored} are not being read. Run `codex config check` for details." - ), - }) + Config::resolve(config_path) } /// Bundle of long-lived guards returned by [`init_tracing`]. @@ -1151,62 +1108,6 @@ mod tests { assert!(plugins_dir.exists()); } - fn rename_finding(var: &str) -> codex_config::Finding { - codex_config::Finding::WillRename { - var: var.to_string(), - v2_name: format!("{var}__X"), - path: "task.worker_count".to_string(), - } - } - - fn ignored_finding(var: &str) -> codex_config::Finding { - codex_config::Finding::NotYetValid { - var: var.to_string(), - v1_name: "CODEX_TASK_WORKER_COUNT".to_string(), - } - } - - #[test] - fn env_notice_is_silent_when_nothing_is_wrong() { - assert_eq!(env_var_notice(&[]), None); - } - - /// One line regardless of how many variables are involved. A per-variable - /// warning would be recurring noise about names that are still correct in - /// this version. - #[test] - fn env_notice_is_a_single_line_that_names_no_variables() { - let findings: Vec<_> = (0..14) - .map(|i| rename_finding(&format!("CODEX_THING_{i}"))) - .collect(); - let notice = env_var_notice(&findings).expect("should produce a notice"); - - assert_eq!(notice.lines().count(), 1, "notice must be one line"); - assert!( - !notice.contains("CODEX_"), - "notice must not name variables: {notice}" - ); - assert!(notice.contains("14"), "notice should carry the count"); - assert!(notice.contains("codex config check")); - } - - #[test] - fn env_notice_distinguishes_renames_from_ignored_variables() { - let renames_only = env_var_notice(&[rename_finding("CODEX_A")]).unwrap(); - assert!(renames_only.contains("renamed in Codex 2.0")); - assert!(!renames_only.contains("not being read")); - - let ignored_only = env_var_notice(&[ignored_finding("CODEX_B__C")]).unwrap(); - assert!(ignored_only.contains("not being read")); - assert!(!ignored_only.contains("renamed in Codex 2.0")); - - let both = env_var_notice(&[rename_finding("CODEX_A"), ignored_finding("CODEX_B__C")]) - .expect("should produce a notice"); - assert!(both.contains("renamed in Codex 2.0")); - assert!(both.contains("not being read")); - assert_eq!(both.lines().count(), 1); - } - /// `resolve_config` backs `codex config check`, which is meant to run /// against a read-only mount as an initContainer. #[test] diff --git a/crates/codex-config/Cargo.toml b/crates/codex-config/Cargo.toml index 24edd3bbf..3211bb2c1 100644 --- a/crates/codex-config/Cargo.toml +++ b/crates/codex-config/Cargo.toml @@ -14,6 +14,7 @@ figment = { version = "0.10", features = ["yaml", "toml", "env"] } serde_json = "1.0" serde_yaml = { workspace = true } anyhow = { workspace = true } +tracing = { workspace = true } [dev-dependencies] figment = { version = "0.10", features = ["test"] } diff --git a/crates/codex-config/src/env_audit.rs b/crates/codex-config/src/env_audit.rs index 65634bae2..54bb12f9a 100644 --- a/crates/codex-config/src/env_audit.rs +++ b/crates/codex-config/src/env_audit.rs @@ -6,9 +6,11 @@ //! not mechanically reversible, which is why v2 switches to `__` between //! levels: `CODEX_RATE_LIMIT__ANONYMOUS_RPS`. //! -//! This module maps a variable name to the setting it was aiming at. It powers -//! two things: `codex config check`, which reports the v2 name for every -//! variable that changes, and a single advisory line at startup. +//! This module maps a variable name to the setting it was aiming at. A name +//! in the old flat form is no longer read, so it is reported as an error with +//! its replacement rather than ignored: a deployment that keeps +//! `CODEX_RATE_LIMIT_ANONYMOUS_RPS` would otherwise silently run with default +//! rate limits. //! //! It also catches plain mistakes. Several variables in the documentation //! today do nothing at all (`CODEX_DATABASE_POSTGRES_USER` instead of @@ -18,7 +20,7 @@ use crate::keys::{KeyRegistry, registry}; use crate::loader::ENV_PREFIX; -use crate::types::Config; +use crate::types::{Config, ConfigError}; use std::collections::{BTreeSet, HashMap}; use std::sync::OnceLock; @@ -91,23 +93,14 @@ pub const REMOVED_VARS: &[(&str, &str, &str)] = &[ /// as advice in v1.44 and raised as errors in v2.0. #[derive(Debug, Clone, PartialEq, Eq)] pub enum Finding { - /// A valid v1 name whose v2 spelling differs. - WillRename { + /// The old flat spelling. No longer read. + Legacy { var: String, - /// The v2 spelling, using `__` between nesting levels. - v2_name: String, - /// The config path both names resolve to. + /// The nested spelling that replaces it. + replacement: String, + /// The config path both forms aimed at. path: String, }, - /// A v2-style name, which this version does not read. - /// - /// Worth reporting loudly: the setting is being ignored right now, and - /// renaming ahead of the upgrade is the way operators get here. - NotYetValid { - var: String, - /// The name this version does read. - v1_name: String, - }, /// Removed in favour of a config key that cannot be derived by /// re-spelling the old name. Removed { @@ -130,19 +123,21 @@ impl Finding { /// The environment variable this finding is about. pub fn var(&self) -> &str { match self { - Finding::WillRename { var, .. } - | Finding::NotYetValid { var, .. } + Finding::Legacy { var, .. } | Finding::Removed { var, .. } | Finding::Unknown { var, .. } => var, } } - /// Whether this finding means a setting is being ignored right now. - pub fn is_ignored_now(&self) -> bool { - matches!( - self, - Finding::NotYetValid { .. } | Finding::Removed { .. } | Finding::Unknown { .. } - ) + /// Whether this must stop startup. + /// + /// True where the operator named a real setting in a form that is no + /// longer read, so continuing would run with a value they did not choose. + /// An unrecognized name is only a warning: another tool may legitimately + /// use the `CODEX_` prefix, and guessing wrong should not take a + /// deployment down. + pub fn is_fatal(&self) -> bool { + matches!(self, Finding::Legacy { .. } | Finding::Removed { .. }) } } @@ -212,37 +207,35 @@ pub fn classify(var: &str, registry: &KeyRegistry) -> Option { }); } - // A `__` anywhere means the operator wrote a v2-style name. + // A `__` anywhere means the operator wrote the current form. If it names a + // real setting there is nothing to say; the loader has already read it. if rest.contains("__") { let path = rest .split("__") .map(|s| s.to_lowercase()) .collect::>() .join("."); - return Some(if registry.contains(&path) { - Finding::NotYetValid { - var: var.to_string(), - v1_name: v1_name_for(&path), - } + return if registry.contains(&path) { + None } else { - Finding::Unknown { + Some(Finding::Unknown { var: var.to_string(), nearest: nearest_path(&path, registry), - } - }); + }) + }; } match resolve_flat(rest, registry) { Some(path) => { - let v2_name = v2_name_for(&path); - // Single-segment settings such as `data_dir` spell the same in - // both schemes; there is nothing for the operator to change. - if v2_name == var { + let replacement = v2_name_for(&path); + // Single-segment settings such as `data_dir` spell the same either + // way, so they are still read and there is nothing to change. + if replacement == var { None } else { - Some(Finding::WillRename { + Some(Finding::Legacy { var: var.to_string(), - v2_name, + replacement, path, }) } @@ -254,6 +247,47 @@ pub fn classify(var: &str, registry: &KeyRegistry) -> Option { } } +/// Fail when the environment names a setting in a form that is no longer read. +/// +/// Every offending variable is reported in one message. An operator with a +/// dozen of them should fix all twelve in one pass, not discover them one +/// restart at a time. +pub fn enforce_env(config: &Config) -> Result, ConfigError> { + let findings = audit_env_with_config(config); + let (fatal, warnings): (Vec<_>, Vec<_>) = findings.iter().partition(|f| f.is_fatal()); + + if fatal.is_empty() { + return Ok(warnings.into_iter().cloned().collect()); + } + + let mut message = String::from("environment variables that are no longer read:\n"); + for finding in &fatal { + match finding { + Finding::Legacy { + var, replacement, .. + } => { + message.push_str(&format!(" {var}\n renamed to {replacement}\n")); + } + Finding::Removed { + var, + replacement, + note, + } => { + message.push_str(&format!( + " {var}\n replaced by {replacement} ({note})\n" + )); + } + _ => {} + } + } + message.push_str( + "\nNesting levels are separated by `__` since Codex 2.0. \ + Run `codex config check` to see this list without starting the server.", + ); + + Err(ConfigError::new(message)) +} + /// Resolve a flat v1 suffix (everything after `CODEX_`) to a config path. fn resolve_flat(rest: &str, registry: &KeyRegistry) -> Option { if let Some(path) = normalized_index(registry).get(&normalize(rest)) { @@ -329,7 +363,8 @@ fn match_segments(segments: &[&str], rest: &str) -> Option> { match_segments(tail, remainder.strip_prefix('_')?) } -/// The v1 spelling of a config path: every separator is a single `_`. +/// The pre-2.0 flat spelling of a config path, where every separator was a +/// single `_`. Kept for describing what an operator must change. pub fn v1_name_for(path: &str) -> String { format!("{ENV_PREFIX}{}", path.replace('.', "_").to_uppercase()) } @@ -426,13 +461,15 @@ mod tests { fn rename(var: &str) -> (String, String) { match classify_one(var) { - Some(Finding::WillRename { v2_name, path, .. }) => (v2_name, path), - other => panic!("expected {var} to be a rename, got {other:?}"), + Some(Finding::Legacy { + replacement, path, .. + }) => (replacement, path), + other => panic!("expected {var} to be a legacy name, got {other:?}"), } } #[test] - fn flat_names_map_to_double_underscore_names() { + fn legacy_flat_names_map_to_double_underscore_names() { for (var, expected_v2, expected_path) in [ ( "CODEX_TASK_WORKER_COUNT", @@ -616,14 +653,27 @@ mod tests { assert!(!NON_CONFIG_VARS.contains(&"CODEX_BIN_VERSION")); } + /// The current spelling is simply read; there is nothing to report. #[test] - fn v2_names_are_reported_as_not_yet_valid() { - match classify_one("CODEX_TASK__WORKER_COUNT") { - Some(Finding::NotYetValid { v1_name, .. }) => { - assert_eq!(v1_name, "CODEX_TASK_WORKER_COUNT"); - } - other => panic!("expected NotYetValid, got {other:?}"), - } + fn nested_names_are_accepted_silently() { + assert_eq!(classify_one("CODEX_TASK__WORKER_COUNT"), None); + assert_eq!(classify_one("CODEX_DATABASE__POSTGRES__HOST"), None); + } + + /// A nested name that does not resolve is still just a warning. + #[test] + fn a_nested_name_for_no_setting_is_unknown() { + assert!(matches!( + classify_one("CODEX_TASK__NOPE"), + Some(Finding::Unknown { .. }) + )); + } + + #[test] + fn legacy_and_removed_are_fatal_but_unknown_is_not() { + assert!(classify_one("CODEX_TASK_WORKER_COUNT").unwrap().is_fatal()); + assert!(classify_one("CODEX_DISABLE_WORKERS").unwrap().is_fatal()); + assert!(!classify_one("CODEX_NOT_A_THING_AT_ALL").unwrap().is_fatal()); } #[test] @@ -764,8 +814,8 @@ mod tests { ]; for var in documented { assert!( - matches!(classify_one(var), Some(Finding::WillRename { .. })), - "{var} should resolve to a rename, got {:?}", + matches!(classify_one(var), Some(Finding::Legacy { .. })), + "{var} should be reported as a legacy spelling, got {:?}", classify_one(var) ); } diff --git a/crates/codex-config/src/lib.rs b/crates/codex-config/src/lib.rs index 45c29061d..f64d7b0ea 100644 --- a/crates/codex-config/src/lib.rs +++ b/crates/codex-config/src/lib.rs @@ -21,8 +21,8 @@ pub use types::{ }; pub use env_audit::{ - Finding, NON_CONFIG_VARS, audit, audit_env, audit_env_with_config, classify, - secret_env_targets, v1_name_for, v2_name_for, + Finding, NON_CONFIG_VARS, REMOVED_VARS, audit, audit_env, audit_env_with_config, classify, + enforce_env, secret_env_targets, v1_name_for, v2_name_for, }; pub use keys::{KeyRegistry, registry}; pub use loader::{ENV_NESTING_SEPARATOR, ENV_PREFIX}; diff --git a/crates/codex-config/src/loader.rs b/crates/codex-config/src/loader.rs index e8c5c2d4e..f62d9788b 100644 --- a/crates/codex-config/src/loader.rs +++ b/crates/codex-config/src/loader.rs @@ -35,6 +35,34 @@ impl Config { /// are a complete configuration on their own, which is what a container /// with no mounted file relies on. pub fn load>(path: P) -> Result { + let config = Self::resolve(path)?; + + // After resolution, not before: unknown keys are ignored by serde so + // extraction always succeeds in the case this catches, and exempting + // the variables a provider's `client_secret_env` names needs the + // resolved config. + for warning in crate::enforce_env(&config)? { + if let crate::Finding::Unknown { var, nearest } = warning { + match nearest { + Some(path) => tracing::warn!( + "{var} is not a Codex setting; did you mean {}?", + crate::v2_name_for(&path) + ), + None => tracing::warn!("{var} is not a Codex setting; ignoring"), + } + } + } + + Ok(config) + } + + /// Resolve configuration without checking the environment for names that + /// are no longer read. + /// + /// `codex config check` uses this so it can report every problem at once + /// instead of dying on the first. Everything that starts a process wants + /// [`Config::load`]. + pub fn resolve>(path: P) -> Result { let path = path.as_ref(); // The same layers twice: once over the defaults to produce the config, @@ -221,12 +249,72 @@ mod tests { }); } - /// v1 spelling must NOT be honoured. It is reported by the validator with - /// its replacement instead of quietly half-working. + /// The old flat spelling is not read, and must not be quietly ignored: + /// a deployment that keeps it would run on a value nobody chose. + #[test] + fn the_old_flat_names_stop_startup() { + Jail::expect_with(|jail| { + jail.set_env("CODEX_APPLICATION_PORT", "4321"); + + let error = Config::load(jail.directory().join("absent.yaml")).unwrap_err(); + let message = format!("{error:#}"); + + assert!(message.contains("CODEX_APPLICATION_PORT"), "{message}"); + assert!( + message.contains("CODEX_APPLICATION__PORT"), + "error must name the replacement: {message}" + ); + Ok(()) + }); + } + + /// `resolve` skips the check so `config check` can report everything at + /// once rather than dying on the first offender. #[test] - fn the_old_flat_names_are_not_read() { + fn resolve_ignores_the_old_names_instead_of_failing() { Jail::expect_with(|jail| { jail.set_env("CODEX_APPLICATION_PORT", "4321"); + + let config = Config::resolve(jail.directory().join("absent.yaml")).unwrap(); + + assert_eq!(config.application.port, 8080, "the old name is not read"); + Ok(()) + }); + } + + /// Every offender in one message: twelve of them should be one fix, not + /// twelve restarts. + #[test] + fn every_offending_variable_is_reported_together() { + Jail::expect_with(|jail| { + jail.set_env("CODEX_APPLICATION_PORT", "1"); + jail.set_env("CODEX_TASK_WORKER_COUNT", "2"); + jail.set_env("CODEX_DISABLE_WORKERS", "true"); + + let error = Config::load(jail.directory().join("absent.yaml")).unwrap_err(); + let message = format!("{error:#}"); + + for var in [ + "CODEX_APPLICATION_PORT", + "CODEX_TASK_WORKER_COUNT", + "CODEX_DISABLE_WORKERS", + ] { + assert!(message.contains(var), "{var} missing from: {message}"); + } + assert!( + message.contains("INVERTED"), + "the inverted replacement must be called out: {message}" + ); + Ok(()) + }); + } + + /// An unrecognized name is a warning, never fatal: another tool may use + /// the same prefix, and guessing wrong should not take a deployment down. + #[test] + fn an_unrecognized_variable_does_not_stop_startup() { + Jail::expect_with(|jail| { + jail.set_env("CODEX_SOMETHING_NOBODY_KNOWS", "x"); let config = Config::load(jail.directory().join("absent.yaml")).unwrap(); assert_eq!(config.application.port, 8080); Ok(()) diff --git a/crates/codex-config/src/types.rs b/crates/codex-config/src/types.rs index 049ffd4c8..fa7ba9e3f 100644 --- a/crates/codex-config/src/types.rs +++ b/crates/codex-config/src/types.rs @@ -347,6 +347,12 @@ const DEFAULT_PLUGINS_SUBDIR: &str = "plugins"; #[derive(Debug)] pub struct ConfigError(String); +impl ConfigError { + pub(crate) fn new(message: impl Into) -> Self { + Self(message.into()) + } +} + impl std::fmt::Display for ConfigError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str(&self.0) diff --git a/src/commands/config.rs b/src/commands/config.rs index cede0c226..184006eb5 100644 --- a/src/commands/config.rs +++ b/src/commands/config.rs @@ -1,8 +1,9 @@ //! `codex config` — inspect configuration without starting the server. //! -//! `check` resolves the configuration exactly as `serve` would, reports every -//! environment variable that this version does not read or that changes name -//! in the next major version, and prints the result with secrets removed. +//! `check` resolves the configuration the way `serve` does, reports every +//! `CODEX_` variable that is not read, and prints the result with secrets +//! removed. Unlike `serve` it lists every problem rather than stopping at the +//! first, which is the whole point of running it before a deployment. //! //! It opens no database connection and writes nothing, so it is safe to run as //! a Kubernetes initContainer ahead of the app container, or as a one-off Job @@ -20,7 +21,7 @@ use codex_config::{Config, Finding, audit_env_with_config, redacted_yaml}; pub enum ConfigSubcommand { /// Validate environment variables and print the resolved configuration Check { - /// Exit non-zero if anything at all was reported, not just errors + /// Also fail on warnings, not just on settings that are no longer read #[arg(long)] strict: bool, @@ -37,13 +38,18 @@ pub fn config_command(config_path: PathBuf, command: ConfigSubcommand) -> Result } fn check(config_path: &Path, strict: bool, quiet: bool) -> Result<()> { + // `resolve` rather than `load`: the point is to list every problem, + // and `load` refuses to start on the first one. let config = resolve_config(config_path)?; let findings = audit_env_with_config(&config); let report = build_report(config_path, &config, &findings, quiet)?; print!("{report}"); - if strict && !findings.is_empty() { + // Match what the server will do: a fatal finding stops it, so `check` + // fails too. `--strict` additionally fails on warnings. + let failed = findings.iter().any(Finding::is_fatal) || (strict && !findings.is_empty()); + if failed { std::process::exit(1); } Ok(()) @@ -74,13 +80,13 @@ fn build_report( } )?; - let renames: Vec<&Finding> = findings + let legacy: Vec<&Finding> = findings .iter() - .filter(|f| matches!(f, Finding::WillRename { .. })) + .filter(|f| matches!(f, Finding::Legacy { .. })) .collect(); - let ignored: Vec<&Finding> = findings + let removed: Vec<&Finding> = findings .iter() - .filter(|f| matches!(f, Finding::NotYetValid { .. })) + .filter(|f| matches!(f, Finding::Removed { .. })) .collect(); let unknown: Vec<&Finding> = findings .iter() @@ -91,34 +97,42 @@ fn build_report( writeln!(out, "\n No environment variable problems found.")?; } - if !renames.is_empty() { + if !legacy.is_empty() { writeln!( out, - "\nEnvironment variables that change name in Codex 2.0 ({}):", - renames.len() + "\nERROR: environment variables that are no longer read ({}):", + legacy.len() )?; - let width = renames.iter().map(|f| f.var().len()).max().unwrap_or(0); - for finding in &renames { - if let Finding::WillRename { var, v2_name, .. } = finding { - writeln!(out, " {var: {v2_name}")?; + let width = legacy.iter().map(|f| f.var().len()).max().unwrap_or(0); + for finding in &legacy { + if let Finding::Legacy { + var, replacement, .. + } = finding + { + writeln!(out, " {var: {replacement}")?; } } writeln!( out, - "\n These names are correct for this version. Do not rename them until you\n \ - upgrade to Codex 2.0: this version does not read the new spelling." + "\n Nesting levels are separated by `__` since Codex 2.0." )?; + writeln!(out, " A single `_` still separates words inside one key.")?; } - if !ignored.is_empty() { + if !removed.is_empty() { writeln!( out, - "\nEnvironment variables that are NOT being read right now ({}):", - ignored.len() + "\nERROR: environment variables that were replaced ({}):", + removed.len() )?; - for finding in &ignored { - if let Finding::NotYetValid { var, v1_name } = finding { - writeln!(out, " {var}\n this version reads {v1_name} instead")?; + for finding in &removed { + if let Finding::Removed { + var, + replacement, + note, + } = finding + { + writeln!(out, " {var}\n use {replacement} ({note})")?; } } } @@ -126,7 +140,7 @@ fn build_report( if !unknown.is_empty() { writeln!( out, - "\nUnrecognized environment variables ({}):", + "\nWarning: unrecognized environment variables ({}):", unknown.len() )?; for finding in &unknown { @@ -135,7 +149,7 @@ fn build_report( Some(path) => writeln!( out, " {var}\n not a Codex setting; did you mean {}?", - codex_config::v1_name_for(path) + codex_config::v2_name_for(path) )?, None => writeln!(out, " {var}\n not a Codex setting; ignored")?, } @@ -157,10 +171,10 @@ fn build_report( mod tests { use super::*; - fn rename(var: &str, v2_name: &str, path: &str) -> Finding { - Finding::WillRename { + fn legacy(var: &str, replacement: &str, path: &str) -> Finding { + Finding::Legacy { var: var.to_string(), - v2_name: v2_name.to_string(), + replacement: replacement.to_string(), path: path.to_string(), } } @@ -176,15 +190,15 @@ mod tests { } #[test] - fn renames_are_listed_with_their_replacement() { + fn legacy_names_are_listed_with_their_replacement() { let text = report( &[ - rename( + legacy( "CODEX_TASK_WORKER_COUNT", "CODEX_TASK__WORKER_COUNT", "task.worker_count", ), - rename( + legacy( "CODEX_APPLICATION_PORT", "CODEX_APPLICATION__PORT", "application.port", @@ -192,45 +206,32 @@ mod tests { ], true, ); - assert!(text.contains("change name in Codex 2.0 (2)")); - assert!(text.contains("CODEX_TASK_WORKER_COUNT")); + assert!(text.contains("no longer read (2)")); assert!(text.contains("CODEX_TASK__WORKER_COUNT")); assert!(text.contains("CODEX_APPLICATION__PORT")); + assert!(text.contains("separated by `__`")); } - /// Renaming early silently drops the setting, so the report has to say so - /// rather than just handing over the new name. + /// The two inverted replacements are the ones worth reading carefully, so + /// the note has to reach the report. #[test] - fn the_report_warns_against_renaming_early() { + fn replaced_variables_carry_their_note() { let text = report( - &[rename( - "CODEX_TASK_WORKER_COUNT", - "CODEX_TASK__WORKER_COUNT", - "task.worker_count", - )], - true, - ); - assert!( - text.contains("Do not rename them until you"), - "missing the do-not-rename-early warning:\n{text}" - ); - } - - #[test] - fn variables_not_being_read_are_called_out() { - let text = report( - &[Finding::NotYetValid { - var: "CODEX_TASK__WORKER_COUNT".to_string(), - v1_name: "CODEX_TASK_WORKER_COUNT".to_string(), + &[Finding::Removed { + var: "CODEX_DISABLE_WORKERS".to_string(), + replacement: "CODEX_TASK__RUN_IN_PROCESS".to_string(), + note: "INVERTED: `DISABLE_WORKERS=true` becomes `RUN_IN_PROCESS=false`".to_string(), }], true, ); - assert!(text.contains("NOT being read right now (1)")); - assert!(text.contains("this version reads CODEX_TASK_WORKER_COUNT instead")); + assert!(text.contains("were replaced (1)")); + assert!(text.contains("CODEX_TASK__RUN_IN_PROCESS")); + assert!(text.contains("INVERTED")); } + /// Unknown names are advisory; the heading must not read as an error. #[test] - fn unknown_variables_suggest_a_v1_name() { + fn unknown_variables_are_a_warning_with_a_suggestion() { let text = report( &[Finding::Unknown { var: "CODEX_DATABASE_POSTGRES_USER".to_string(), @@ -238,11 +239,8 @@ mod tests { }], true, ); - assert!(text.contains("Unrecognized environment variables (1)")); - assert!( - text.contains("did you mean CODEX_DATABASE_POSTGRES_USERNAME?"), - "suggestion should use this version's spelling:\n{text}" - ); + assert!(text.contains("Warning: unrecognized")); + assert!(text.contains("did you mean CODEX_DATABASE__POSTGRES__USERNAME?")); } #[test] @@ -259,8 +257,7 @@ mod tests { #[test] fn a_clean_environment_reports_nothing_to_fix() { - let text = report(&[], true); - assert!(text.contains("No environment variable problems found.")); + assert!(report(&[], true).contains("No environment variable problems found.")); } #[test] From c5297dd21f54ea3d2c9044f04246dad2c58c48b7 Mon Sep 17 00:00:00 2001 From: Sylvain Cau Date: Sat, 15 Aug 2026 17:34:26 -0700 Subject: [PATCH 06/11] feat(config)!: stop writing a config file at startup, add `codex config init` Starting with no config file wrote `Config::default()` to disk. That was already a poor introduction, since serializing the live defaults produces a file with no comments explaining any of it, but it was also a way to leak secrets: the defaults used to be read from the environment, so a container started once with a database password in its environment wrote that password into the generated YAML in plaintext, where it stayed. Loading now leaves no trace. A missing file is not an error and is not created, because defaults plus the environment are already a complete configuration, which is what a container with nothing mounted relies on. `codex config init` writes a commented starter template when an operator asks for one, and refuses to overwrite without `--force` so a stray invocation cannot clobber a tuned production config. The template documents the precedence chain, the `.local` overlay, and the `__` environment convention, and leaves optional sections commented so that nothing in it counts as an explicit setting. That last part matters for the paths: an uncommented `thumbnail_dir` would pin the path and stop it following `data_dir`. Two tests cover it: loading creates neither the file nor its parent directory, and a password in the environment never reaches disk. The template itself is parsed in a test, since a starter that does not load would hand the operator a broken server, and checked for a mention of every config section so a new one cannot go undocumented. `load_config` returns just the config now; the "did I create it?" flag it used to return has nothing left to report. --- Cargo.lock | 1 + config/codex.example.yaml | 171 ++++++++++++++++++++++++++++ crates/codex-cli-common/Cargo.toml | 1 + crates/codex-cli-common/src/lib.rs | 71 +++++++----- crates/codex-config/src/lib.rs | 2 +- crates/codex-config/src/loader.rs | 98 ++++++++++++++++ src/commands/config.rs | 15 ++- src/commands/copy.rs | 8 +- src/commands/export.rs | 2 +- src/commands/import.rs | 6 +- src/commands/migrate.rs | 2 +- src/commands/serve.rs | 5 +- src/commands/tasks.rs | 2 +- src/commands/wait_for_migrations.rs | 2 +- src/commands/worker.rs | 2 +- 15 files changed, 340 insertions(+), 48 deletions(-) create mode 100644 config/codex.example.yaml diff --git a/Cargo.lock b/Cargo.lock index d22b32b99..1d3b42845 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -939,6 +939,7 @@ dependencies = [ "codex-events", "codex-services", "codex-tasks", + "figment", "sea-orm", "serial_test", "tempfile", diff --git a/config/codex.example.yaml b/config/codex.example.yaml new file mode 100644 index 000000000..de8f06e4b --- /dev/null +++ b/config/codex.example.yaml @@ -0,0 +1,171 @@ +# Codex configuration +# +# Every key below is optional: Codex starts with no config file at all, and +# anything absent here falls back to the default shown in the comment. +# +# Precedence, lowest to highest: +# +# 1. built-in defaults +# 2. this file +# 3. codex.local.yaml (a sibling file, if present) +# 4. CODEX_* environment variables +# +# `codex.local.yaml` is the place for secrets and per-host overrides. It merges +# key by key, so it only needs the handful you want to change. Lists are the +# exception: a list in the overlay replaces the base list rather than adding +# to it. +# +# Environment variables separate nesting levels with a double underscore and +# words within one key with a single underscore: +# +# rate_limit.anonymous_rps -> CODEX_RATE_LIMIT__ANONYMOUS_RPS +# database.postgres.host -> CODEX_DATABASE__POSTGRES__HOST +# +# Run `codex config check` to validate this file and see the resolved result +# with secrets redacted. + +# Base directory for everything Codex writes. Each path below defaults to a +# subdirectory of it, so setting this one key moves them all together. +data_dir: data + +database: + # sqlite (default) or postgres + db_type: sqlite + + sqlite: + # Defaults to /codex.db + # path: data/codex.db + pragmas: + journal_mode: WAL + foreign_keys: "ON" + # max_connections: 64 + # min_connections: 2 + + # postgres: + # host: localhost + # port: 5432 + # username: codex + # password: codex # prefer codex.local.yaml or the environment + # database_name: codex + # # Per process, not per deployment. Every replica, worker and job opens + # # its own pool against the server's shared max_connections. + # max_connections: 25 + + # Apply pending migrations at startup. Set false when a separate Job or init + # container owns them; Codex then waits for the schema instead of applying it. + # run_migrations: true + # migration_wait_timeout_secs: 300 + # migration_wait_interval_secs: 2 + +application: + host: 0.0.0.0 + port: 8080 + # Public URL, used for OIDC redirects and email links. Without it Codex + # falls back to http://{host}:{port}, which is wrong behind a proxy. + # base_url: https://codex.example.com + +auth: + # Generate with: openssl rand -base64 32 + jwt_secret: INSECURE_DEFAULT_SECRET_CHANGE_IN_PRODUCTION + # jwt_expiry_hours: 24 + # refresh_token_enabled: true + # Send `Secure` on auth cookies. Off by default so plain-HTTP development + # works; turn it on anywhere TLS is terminated. + # cookie_secure: true + + # oidc: + # enabled: true + # providers: + # authentik: + # display_name: Authentik + # issuer_url: https://idp.example.com/application/o/codex/ + # client_id: codex + # # Either the secret itself, or the name of a variable holding it: + # # client_secret: ... + # client_secret_env: CODEX_OIDC_AUTHENTIK_SECRET + # role_mapping: + # admin: [codex-admins] + +logging: + # error | warn | info | debug | trace + level: info + console: true + # file: /var/log/codex/codex.log + +# api: +# # base_path: /api/v1 +# # enable_api_docs: false +# # cors_enabled: true +# # cors_origins: ["*"] + +task: + # Workers per process. + worker_count: 2 + # Run workers in this process. Set false for a web-only replica in a + # deployment where workers have their own pods. + # run_in_process: true + +# scanner: +# # max_concurrent_scans: 2 + +scheduler: + # IANA timezone for cron schedules. + timezone: UTC + +# files: +# # All default to subdirectories of data_dir. +# # thumbnail_dir: data/thumbnails +# # uploads_dir: data/uploads +# # plugins_dir: data/plugins + +# images: +# # Concurrent decode/resize/render operations. Each one holds an uncompressed +# # bitmap, so this bounds peak image memory. +# # decode_concurrency: 3 + +# pdf: +# # render_dpi: 150 +# # jpeg_quality: 85 +# # cache_dir: data/cache + +# pdf_handle_cache: +# # enabled: true +# # capacity: 256 + +# plugins: +# # Extra executables plugins may spawn, on top of the built-in runtimes +# # (node, npx, python, python3, uv, uvx). +# # allowed_commands: [deno] + +# rate_limit: +# # enabled: true +# # anonymous_rps: 10 +# # authenticated_rps: 50 + +# komga_api: +# # Komga-compatible API for third-party clients. Off by default. +# # enabled: false +# # prefix: komga + +# koreader_api: +# # enabled: false + +# email: +# smtp_host: localhost +# smtp_port: 587 +# smtp_username: codex +# smtp_password: ... # prefer codex.local.yaml or the environment +# smtp_from_email: noreply@example.com + +# observability: +# enabled: true +# service_name: codex +# otlp: +# endpoint: http://collector:4317 +# protocol: grpc # grpc | http/protobuf | http/json +# headers: {} +# traces: +# enabled: true +# sample_ratio: 1.0 +# metrics: +# enabled: true diff --git a/crates/codex-cli-common/Cargo.toml b/crates/codex-cli-common/Cargo.toml index b708ef1b8..ad97d2e51 100644 --- a/crates/codex-cli-common/Cargo.toml +++ b/crates/codex-cli-common/Cargo.toml @@ -34,6 +34,7 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"] } tracing-opentelemetry = { version = "0.33", optional = true } [dev-dependencies] +figment = { version = "0.10", features = ["test"] } tempfile = { workspace = true } serial_test = { workspace = true } codex-db = { workspace = true, features = ["test-utils"] } diff --git a/crates/codex-cli-common/src/lib.rs b/crates/codex-cli-common/src/lib.rs index f6ce3280f..a0dd13fe7 100644 --- a/crates/codex-cli-common/src/lib.rs +++ b/crates/codex-cli-common/src/lib.rs @@ -80,28 +80,17 @@ pub fn ensure_data_directories(config: &Config) -> anyhow::Result<()> { Ok(()) } -/// Load and apply configuration -pub fn load_config(config_path: PathBuf) -> anyhow::Result<(Config, bool)> { - // Ensure config file parent directory exists - ensure_parent_dir_exists(&config_path)?; - - // Check if config file exists, if not create a default one - let config_created = if !config_path.exists() { - println!( - "Config file not found at {:?}, creating default configuration...", - config_path - ); - let default_config = Config::default(); - default_config.to_file(&config_path)?; - println!("Default config file created at {:?}", config_path); - true - } else { - false - }; - - let config = Config::load(&config_path)?; - - Ok((config, config_created)) +/// Load configuration for a process that is about to run. +/// +/// A missing file is not an error and is not created: defaults plus the +/// environment are a complete configuration, which is what a container with +/// nothing mounted relies on. Codex used to write `Config::default()` to disk +/// here instead, which produced an uncommented dump and, because the defaults +/// were read from the environment at the time, captured whatever secrets were +/// set on that first boot. `codex config init` writes a commented template +/// when an operator actually asks for one. +pub fn load_config(config_path: PathBuf) -> anyhow::Result { + Config::load(&config_path) } /// Resolve configuration for inspection: no file is created, and an @@ -1133,22 +1122,44 @@ mod tests { assert_eq!(config.application.port, 9123); } + /// Loading must leave no trace. Codex used to write `Config::default()` + /// here, which produced an uncommented dump and, because the defaults were + /// read from the environment, captured whatever secrets were set on that + /// first boot. #[test] - fn test_load_config_creates_parent_directory() { + fn load_config_does_not_create_anything() { let temp_dir = TempDir::new().unwrap(); let config_path = temp_dir.path().join("config").join("codex.yaml"); - assert!(!config_path.parent().unwrap().exists()); + let config = load_config(config_path.clone()).unwrap(); - let (config, created) = load_config(config_path.clone()).unwrap(); - - assert!(config_path.parent().unwrap().exists()); - assert!(config_path.exists()); - assert!(created); - // Verify it's a valid config + assert!(!config_path.exists(), "no config file should be written"); + assert!( + !config_path.parent().unwrap().exists(), + "no directory should be created either" + ); assert!(!config.application.host.is_empty()); } + /// The specific leak the change closes. + // `figment::Error` is large and every `Jail` closure returns it. + #[allow(clippy::result_large_err)] + #[test] + fn a_secret_in_the_environment_is_never_written_to_disk() { + figment::Jail::expect_with(|jail| { + let config_path = jail.directory().join("codex.yaml"); + jail.set_env("CODEX_DATABASE__POSTGRES__PASSWORD", "super-secret"); + + load_config(config_path.clone()).unwrap(); + + assert!( + !config_path.exists(), + "startup must not serialize an env-provided secret to disk" + ); + Ok(()) + }); + } + #[tokio::test] async fn test_get_worker_count_from_config() { let task_config = TaskConfig { diff --git a/crates/codex-config/src/lib.rs b/crates/codex-config/src/lib.rs index f64d7b0ea..39a44c850 100644 --- a/crates/codex-config/src/lib.rs +++ b/crates/codex-config/src/lib.rs @@ -25,5 +25,5 @@ pub use env_audit::{ enforce_env, secret_env_targets, v1_name_for, v2_name_for, }; pub use keys::{KeyRegistry, registry}; -pub use loader::{ENV_NESTING_SEPARATOR, ENV_PREFIX}; +pub use loader::{ENV_NESTING_SEPARATOR, ENV_PREFIX, STARTER_CONFIG_YAML, write_starter_config}; pub use redact::{REDACTED, UNSET, redacted_value, redacted_yaml}; diff --git a/crates/codex-config/src/loader.rs b/crates/codex-config/src/loader.rs index f62d9788b..484db72d6 100644 --- a/crates/codex-config/src/loader.rs +++ b/crates/codex-config/src/loader.rs @@ -103,6 +103,38 @@ impl Config { } } +/// The commented starter shipped with the binary, written by `codex config +/// init`. +/// +/// A template rather than a dump of `Config::default()`. Serializing the live +/// defaults produced a file with no comments, and, because the defaults used +/// to be read from the environment, it captured whatever was set at the +/// moment of first boot. A container started once with +/// `CODEX_DATABASE__POSTGRES__PASSWORD` wrote that password into the generated +/// YAML in plaintext. +pub const STARTER_CONFIG_YAML: &str = include_str!("../../../config/codex.example.yaml"); + +/// Write [`STARTER_CONFIG_YAML`] to `path`, creating parent directories. +/// +/// Refuses to overwrite unless `force`, so a stray `config init` cannot clobber +/// a tuned production config. +pub fn write_starter_config(path: &Path, force: bool) -> Result<()> { + if path.exists() && !force { + anyhow::bail!( + "refusing to overwrite the existing config at {}; pass --force to replace it", + path.display() + ); + } + if let Some(parent) = path.parent() + && !parent.as_os_str().is_empty() + { + fs::create_dir_all(parent).with_context(|| format!("creating {}", parent.display()))?; + } + fs::write(path, STARTER_CONFIG_YAML) + .with_context(|| format!("writing starter config to {}", path.display()))?; + Ok(()) +} + /// Stack the file, overlay and environment layers onto `base`. fn layers(path: &Path, base: Figment) -> Figment { let mut figment = base; @@ -869,6 +901,72 @@ database: assert!(!config.auth.cookie_secure(), "Secure was off by default"); } + // ---- starter template ---- + + /// The template must parse, or `config init` hands the operator a file + /// that stops the server. + #[test] + fn the_starter_template_is_a_valid_config() { + Jail::expect_with(|jail| { + jail.create_file("codex.yaml", STARTER_CONFIG_YAML)?; + let config = Config::load(jail.directory().join("codex.yaml")).unwrap(); + assert_eq!(config.application.port, 8080); + assert_eq!(config.scheduler.timezone, "UTC"); + Ok(()) + }); + } + + /// It documents the surface, so a new section that never reaches it is a + /// section operators cannot discover. + #[test] + fn the_starter_template_mentions_every_section() { + for section in [ + "database:", + "application:", + "auth:", + "logging:", + "api:", + "task:", + "scanner:", + "scheduler:", + "files:", + "images:", + "pdf:", + "plugins:", + "rate_limit:", + "komga_api:", + "koreader_api:", + "email:", + "observability:", + ] { + assert!( + STARTER_CONFIG_YAML.contains(section), + "the starter template does not mention `{section}`" + ); + } + } + + #[test] + fn writing_the_starter_creates_parents_and_refuses_to_clobber() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("nested").join("codex.yaml"); + + write_starter_config(&path, false).unwrap(); + assert!(path.exists()); + + std::fs::write(&path, "application:\n port: 1234\n").unwrap(); + let error = write_starter_config(&path, false).unwrap_err(); + assert!(error.to_string().contains("--force"), "{error}"); + assert!(std::fs::read_to_string(&path).unwrap().contains("1234")); + + write_starter_config(&path, true).unwrap(); + assert!( + std::fs::read_to_string(&path) + .unwrap() + .contains("Codex configuration") + ); + } + #[test] fn overlay_path_gets_a_local_infix() { assert_eq!( diff --git a/src/commands/config.rs b/src/commands/config.rs index 184006eb5..02f739729 100644 --- a/src/commands/config.rs +++ b/src/commands/config.rs @@ -14,7 +14,7 @@ use clap::Subcommand; use std::path::{Path, PathBuf}; use codex_cli_common::resolve_config; -use codex_config::{Config, Finding, audit_env_with_config, redacted_yaml}; +use codex_config::{Config, Finding, audit_env_with_config, redacted_yaml, write_starter_config}; /// Configuration subcommands #[derive(Subcommand, Debug)] @@ -29,11 +29,24 @@ pub enum ConfigSubcommand { #[arg(short, long)] quiet: bool, }, + + /// Write a commented starter configuration file + Init { + /// Replace an existing file + #[arg(long)] + force: bool, + }, } pub fn config_command(config_path: PathBuf, command: ConfigSubcommand) -> Result<()> { match command { ConfigSubcommand::Check { strict, quiet } => check(&config_path, strict, quiet), + ConfigSubcommand::Init { force } => { + write_starter_config(&config_path, force)?; + println!("Wrote a starter configuration to {}", config_path.display()); + println!("Edit it, then run `codex config check` to validate."); + Ok(()) + } } } diff --git a/src/commands/copy.rs b/src/commands/copy.rs index 965b6afaa..c8bdb9216 100644 --- a/src/commands/copy.rs +++ b/src/commands/copy.rs @@ -27,7 +27,7 @@ pub async fn copy_command( full_verification: bool, ) -> Result<()> { // Local config: used for tracing and as the fallback for an omitted side. - let (local_config, _created) = load_config(config_path.clone())?; + let local_config = load_config(config_path.clone())?; let _tracing = init_tracing(&local_config)?; let source_explicit = resolve_side( @@ -139,7 +139,7 @@ fn resolve_side( return Ok(Some(database_config_from_url(&u)?)); } if let Some(path) = config_file { - let (config, _created) = load_config(path.to_path_buf())?; + let config = load_config(path.to_path_buf())?; return Ok(Some(config.database)); } Ok(None) @@ -197,7 +197,7 @@ files: // Seed the source. { - let (config, _) = load_config(src_cfg.clone()).unwrap(); + let config = load_config(src_cfg.clone()).unwrap(); let db = Database::new(&config.database).await.unwrap(); db.run_migrations().await.unwrap(); db.create_library("Manga", "/lib", codex_db::ScanningStrategy::Default) @@ -220,7 +220,7 @@ files: .await .expect("copy should succeed"); - let (config, _) = load_config(tgt_cfg).unwrap(); + let config = load_config(tgt_cfg).unwrap(); let db = Database::new(&config.database).await.unwrap(); let libs = db.list_libraries().await.unwrap(); assert_eq!(libs.len(), 1); diff --git a/src/commands/export.rs b/src/commands/export.rs index a9dd17fc7..e7f7982aa 100644 --- a/src/commands/export.rs +++ b/src/commands/export.rs @@ -20,7 +20,7 @@ pub async fn export_command( no_plugins: bool, progress: bool, ) -> Result<()> { - let (config, _created) = load_config(config_path.clone())?; + let config = load_config(config_path.clone())?; let _tracing = init_tracing(&config)?; info!("Loading configuration from {:?}", config_path); diff --git a/src/commands/import.rs b/src/commands/import.rs index a267542c4..c9d11f0fc 100644 --- a/src/commands/import.rs +++ b/src/commands/import.rs @@ -20,7 +20,7 @@ pub async fn import_command( no_verify: bool, full_verification: bool, ) -> Result<()> { - let (config, _created) = load_config(config_path.clone())?; + let config = load_config(config_path.clone())?; let _tracing = init_tracing(&config)?; info!("Loading configuration from {:?}", config_path); @@ -162,7 +162,7 @@ files: } async fn seed_library(config_path: &Path, name: &str) { - let (config, _) = load_config(config_path.to_path_buf()).unwrap(); + let config = load_config(config_path.to_path_buf()).unwrap(); let db = Database::new(&config.database).await.unwrap(); db.run_migrations().await.unwrap(); db.create_library(name, "/lib", codex_db::ScanningStrategy::Default) @@ -171,7 +171,7 @@ files: } async fn library_names(config_path: &Path) -> Vec { - let (config, _) = load_config(config_path.to_path_buf()).unwrap(); + let config = load_config(config_path.to_path_buf()).unwrap(); let db = Database::new(&config.database).await.unwrap(); db.list_libraries() .await diff --git a/src/commands/migrate.rs b/src/commands/migrate.rs index 27b9f3a0d..991a4cb9e 100644 --- a/src/commands/migrate.rs +++ b/src/commands/migrate.rs @@ -6,7 +6,7 @@ use tracing::info; /// Migrate command handler - runs database migrations and exits pub async fn migrate_command(config_path: PathBuf) -> Result<()> { // Load configuration - let (config, _config_created) = load_config(config_path.clone())?; + let config = load_config(config_path.clone())?; // Initialize tracing with config (composes fmt + optional OTel layer) let _tracing_handles = init_tracing(&config)?; diff --git a/src/commands/serve.rs b/src/commands/serve.rs index 825ff5195..fa6d62ff7 100644 --- a/src/commands/serve.rs +++ b/src/commands/serve.rs @@ -13,7 +13,7 @@ use tracing::info; /// Serve command handler - starts the media server pub async fn serve_command(config_path: PathBuf) -> anyhow::Result<()> { // Load configuration - let (config, config_created) = load_config(config_path.clone())?; + let config = load_config(config_path.clone())?; // Initialize tracing with config (composes fmt + optional OTel layer) let tracing_handles = init_tracing(&config)?; @@ -24,9 +24,6 @@ pub async fn serve_command(config_path: PathBuf) -> anyhow::Result<()> { tracing_handles.observability.metrics_enabled(), ); - if config_created { - info!("Created default configuration file"); - } info!("Loading configuration from {:?}", config_path); info!("Configuration loaded successfully"); diff --git a/src/commands/tasks.rs b/src/commands/tasks.rs index f4ed8b535..e3c758242 100644 --- a/src/commands/tasks.rs +++ b/src/commands/tasks.rs @@ -81,7 +81,7 @@ pub enum TasksSubcommand { /// Main task command handler - routes to specific subcommands pub async fn tasks_command(config_path: PathBuf, subcommand: TasksSubcommand) -> Result<()> { // Load configuration and initialize database - let (config, _) = load_config(config_path)?; + let config = load_config(config_path)?; let db = init_database(&config).await?; let conn = db.sea_orm_connection(); diff --git a/src/commands/wait_for_migrations.rs b/src/commands/wait_for_migrations.rs index bf6ee696e..133a23910 100644 --- a/src/commands/wait_for_migrations.rs +++ b/src/commands/wait_for_migrations.rs @@ -12,7 +12,7 @@ pub async fn wait_for_migrations_command( check_interval_seconds: Option, ) -> Result<()> { // Load configuration - let (config, _config_created) = load_config(config_path.clone())?; + let config = load_config(config_path.clone())?; // Initialize tracing with config (composes fmt + optional OTel layer) let _tracing_handles = init_tracing(&config)?; diff --git a/src/commands/worker.rs b/src/commands/worker.rs index 9ce28a209..fa18146af 100644 --- a/src/commands/worker.rs +++ b/src/commands/worker.rs @@ -12,7 +12,7 @@ use tracing::info; /// Worker command handler - starts task workers without web server pub async fn worker_command(config_path: PathBuf) -> anyhow::Result<()> { // Load configuration - let (config, _config_created) = load_config(config_path.clone())?; + let config = load_config(config_path.clone())?; // Initialize tracing with config (composes fmt + optional OTel layer) let tracing_handles = init_tracing(&config)?; From 3dffa9f5c13de55f85bc150ebd622f198960a571 Mon Sep 17 00:00:00 2001 From: Sylvain Cau Date: Sat, 15 Aug 2026 17:57:23 -0700 Subject: [PATCH 07/11] feat(db)!: make PostgreSQL TLS a real setting and stop corrupting URLs on odd passwords `ssl_mode` appeared in two YAML examples, an environment example and a Security Best Practices entry recommending `verify-full` in production, while no such field existed. Operators following that advice believed they had certificate verification and had none. The 1.44 docs corrected the claim and pointed at the libpq variables, which do work; this adds the setting the docs had been describing all along. database.postgres.ssl_mode disable|allow|prefer|require|verify-ca|verify-full database.postgres.ssl_root_cert database.postgres.ssl_client_cert database.postgres.ssl_client_key Leaving `ssl_mode` unset appends nothing to the connection URL, so the driver resolves it as before and `PGSSLMODE` keeps working for deployments configured that way. Setting it here wins. The default is deliberately unchanged. The driver resolves to `prefer`, which encrypts opportunistically, accepts any certificate and falls back to plaintext, all silently. Tightening that to `require` would break every deployment whose PostgreSQL has no TLS configured, which for a self-hosted product is most of them, and it would break them at the same upgrade that already renames every environment variable. The docs say plainly what `prefer` does and does not protect against. Building the URL properly also fixes a bug that had nothing to do with TLS: the connection string was assembled by interpolating the username and password directly, so a password containing `@`, `/`, `:`, `?` or `#` produced a URL pointing somewhere else entirely. Credentials are now percent-encoded, with a narrower escape set for query values so a certificate path stays readable. Two tests that asserted `ssl_mode` was not a real key have been inverted, which is the point of the change. --- Cargo.lock | 1 + config/codex.example.yaml | 5 ++ crates/codex-config/src/env_audit.rs | 21 +++-- crates/codex-config/src/keys.rs | 11 ++- crates/codex-config/src/lib.rs | 4 +- crates/codex-config/src/types.rs | 65 ++++++++++++++ crates/codex-db/Cargo.toml | 3 + crates/codex-db/src/connection.rs | 122 +++++++++++++++++++++++++-- docs/docs/configuration.md | 66 ++++++--------- docs/docs/deployment/database.md | 48 +++++------ 10 files changed, 264 insertions(+), 82 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1d3b42845..c9091796d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -977,6 +977,7 @@ dependencies = [ "codex-utils", "log", "migration", + "percent-encoding", "rand 0.10.0", "sea-orm", "sea-orm-migration", diff --git a/config/codex.example.yaml b/config/codex.example.yaml index de8f06e4b..c119e4a9a 100644 --- a/config/codex.example.yaml +++ b/config/codex.example.yaml @@ -47,6 +47,11 @@ database: # username: codex # password: codex # prefer codex.local.yaml or the environment # database_name: codex + # # TLS. Unset leaves the driver default (`prefer`), which encrypts when + # # the server offers it but accepts any certificate and falls back to + # # plaintext, both silently. Use verify-full for anything remote. + # ssl_mode: verify-full # disable|allow|prefer|require|verify-ca|verify-full + # ssl_root_cert: /etc/ssl/certs/postgres-ca.crt # # Per process, not per deployment. Every replica, worker and job opens # # its own pool against the server's shared max_connections. # max_connections: 25 diff --git a/crates/codex-config/src/env_audit.rs b/crates/codex-config/src/env_audit.rs index 54bb12f9a..0d3a34bc2 100644 --- a/crates/codex-config/src/env_audit.rs +++ b/crates/codex-config/src/env_audit.rs @@ -699,14 +699,14 @@ mod tests { } } - /// Documented today but backed by no field at all. + /// Documented at some point but backed by no field. + /// + /// `CODEX_DATABASE_POSTGRES_SSL_MODE` is deliberately absent: it was + /// documented without existing, and now it exists, so it resolves as an + /// ordinary legacy spelling rather than a typo. #[test] fn documented_but_nonexistent_settings_are_flagged() { - for var in [ - "CODEX_DATABASE_POSTGRES_SSL_MODE", - "CODEX_PLUGINS_LOG_LEVEL", - "CODEX_THUMBNAIL_CACHE_DIR", - ] { + for var in ["CODEX_PLUGINS_LOG_LEVEL", "CODEX_THUMBNAIL_CACHE_DIR"] { assert!( matches!(classify_one(var), Some(Finding::Unknown { .. })), "{var} should be Unknown, got {:?}", @@ -715,6 +715,15 @@ mod tests { } } + /// Documented in 1.x while no such field existed. It exists now, so the + /// old spelling must resolve to the new key rather than look like a typo. + #[test] + fn the_postgres_tls_mode_now_resolves() { + let (v2, path) = rename("CODEX_DATABASE_POSTGRES_SSL_MODE"); + assert_eq!(v2, "CODEX_DATABASE__POSTGRES__SSL_MODE"); + assert_eq!(path, "database.postgres.ssl_mode"); + } + #[test] fn nonsense_names_suggest_nothing() { match classify_one("CODEX_TOTALLY_UNRELATED_THING_XYZ") { diff --git a/crates/codex-config/src/keys.rs b/crates/codex-config/src/keys.rs index 04a42f35a..d36a62677 100644 --- a/crates/codex-config/src/keys.rs +++ b/crates/codex-config/src/keys.rs @@ -141,7 +141,13 @@ fn walk(value: &Value, path: &str, out: &mut KeyRegistry) { fn key_probe() -> Config { let mut config = Config::default(); - config.database.postgres = Some(PostgresConfig::default()); + config.database.postgres = Some(PostgresConfig { + ssl_mode: Some(crate::PgSslMode::Prefer), + ssl_root_cert: Some(String::new()), + ssl_client_cert: Some(String::new()), + ssl_client_key: Some(String::new()), + ..PostgresConfig::default() + }); config.database.sqlite = Some(SQLiteConfig { pragmas: Some(probe_string_map()), ..SQLiteConfig::default() @@ -264,6 +270,8 @@ mod tests { // Carries `skip_serializing_if`, so it vanishes from the probe // rather than showing up as `null`; the drift tests cannot see it. "auth.cookie_secure", + "database.postgres.ssl_mode", + "database.postgres.ssl_root_cert", ] { assert!(registry.contains(path), "missing `{path}`"); } @@ -318,7 +326,6 @@ mod tests { #[test] fn unknown_paths_are_rejected() { let registry = registry(); - assert!(!registry.contains("database.postgres.ssl_mode")); assert!(!registry.contains("plugins.log_level")); assert!(!registry.contains("task.worker.count")); } diff --git a/crates/codex-config/src/lib.rs b/crates/codex-config/src/lib.rs index 39a44c850..0d78eff7e 100644 --- a/crates/codex-config/src/lib.rs +++ b/crates/codex-config/src/lib.rs @@ -16,8 +16,8 @@ pub use types::{ EmailConfig, FilesConfig, ImagesConfig, KomgaApiConfig, KoreaderApiConfig, LoggingConfig, ObservabilityBrowserConfig, ObservabilityConfig, ObservabilityMetricsConfig, ObservabilityTracesConfig, OidcConfig, OidcDefaultRole, OidcProviderConfig, OtlpConfig, - OtlpProtocol, PdfConfig, PdfHandleCacheConfig, PluginsConfig, PostgresConfig, RateLimitConfig, - SQLiteConfig, ScannerConfig, SchedulerConfig, TaskConfig, + OtlpProtocol, PdfConfig, PdfHandleCacheConfig, PgSslMode, PluginsConfig, PostgresConfig, + RateLimitConfig, SQLiteConfig, ScannerConfig, SchedulerConfig, TaskConfig, }; pub use env_audit::{ diff --git a/crates/codex-config/src/types.rs b/crates/codex-config/src/types.rs index fa7ba9e3f..dd3a5184c 100644 --- a/crates/codex-config/src/types.rs +++ b/crates/codex-config/src/types.rs @@ -713,6 +713,42 @@ pub enum DatabaseType { SQLite, } +/// How much protection to demand on the PostgreSQL connection. +/// +/// The names match libpq, and so do the semantics. Note the gap between +/// `require` and `verify-ca`: `require` encrypts but accepts any certificate, +/// which stops passive capture and not an active attacker. +#[derive(Debug, Serialize, Deserialize, Clone, Copy, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +pub enum PgSslMode { + /// Never use TLS. + Disable, + /// Use TLS only if the server insists. + Allow, + /// Use TLS if offered, fall back to plaintext, verify nothing. + Prefer, + /// Require TLS, but accept any certificate. + Require, + /// Require TLS and verify the certificate against the CA. + VerifyCa, + /// Require TLS and verify both the certificate and the hostname. + VerifyFull, +} + +impl PgSslMode { + /// The libpq spelling, which is what goes in the connection URL. + pub fn as_str(&self) -> &'static str { + match self { + PgSslMode::Disable => "disable", + PgSslMode::Allow => "allow", + PgSslMode::Prefer => "prefer", + PgSslMode::Require => "require", + PgSslMode::VerifyCa => "verify-ca", + PgSslMode::VerifyFull => "verify-full", + } + } +} + #[derive(Debug, Serialize, Deserialize, Clone)] #[serde(default)] pub struct PostgresConfig { @@ -722,6 +758,31 @@ pub struct PostgresConfig { pub password: String, pub database_name: String, + /// TLS mode for this connection. + /// + /// Unset leaves the driver's own default in place, which is `prefer`: + /// encrypt when the server offers it, accept any certificate, and fall + /// back to plaintext otherwise, silently in both cases. That is adequate + /// on a private network and not much else. Setting it here also overrides + /// `PGSSLMODE`, which remains the fallback for deployments configured + /// before this setting existed. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ssl_mode: Option, + + /// CA certificate used to verify the server, required by `verify-ca` and + /// `verify-full` unless the system trust store already has it. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[serde(deserialize_with = "crate::de::optional_string")] + pub ssl_root_cert: Option, + + /// Client certificate and key, for mutual TLS. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[serde(deserialize_with = "crate::de::optional_string")] + pub ssl_client_cert: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[serde(deserialize_with = "crate::de::optional_string")] + pub ssl_client_key: Option, + // Connection Pool Settings /// Maximum number of connections in the pool (default: 25) /// @@ -783,6 +844,10 @@ impl Default for PostgresConfig { username: "codex".to_string(), password: "codex".to_string(), database_name: "codex".to_string(), + ssl_mode: None, + ssl_root_cert: None, + ssl_client_cert: None, + ssl_client_key: None, // Pool settings - PostgreSQL can handle more concurrent connections max_connections: 25, min_connections: 2, diff --git a/crates/codex-db/Cargo.toml b/crates/codex-db/Cargo.toml index 956bf4433..e126333f0 100644 --- a/crates/codex-db/Cargo.toml +++ b/crates/codex-db/Cargo.toml @@ -48,6 +48,9 @@ migration = { path = "../../migration" } # Repository-level helpers. serde_json = "1.0" rand = "0.10" +# Credentials go into a connection URL, so they must be percent-encoded. +percent-encoding = "2" + # Used by `connection::Database` to set sqlx logging level on connect. log = "0.4" # Pulled in optionally for `test-utils` so test_helpers can mint temp diff --git a/crates/codex-db/src/connection.rs b/crates/codex-db/src/connection.rs index 6b9280169..92ae2f085 100644 --- a/crates/codex-db/src/connection.rs +++ b/crates/codex-db/src/connection.rs @@ -27,6 +27,70 @@ pub struct Database { conn: DatabaseConnection, } +/// Characters that must be escaped inside the userinfo part of a URL. +/// +/// A password is arbitrary text, and `@`, `/`, `:`, `?` and `#` all mean +/// something to a URL parser. Interpolating one directly, as this used to, +/// silently produced a connection string pointing somewhere else. +const USERINFO_ESCAPE: &percent_encoding::AsciiSet = &percent_encoding::NON_ALPHANUMERIC + .remove(b'-') + .remove(b'.') + .remove(b'_') + .remove(b'~'); + +/// Characters escaped inside a query value. Narrower than the userinfo set: +/// `-._~`, `/` and `:` are all legal here, and escaping them turns a readable +/// certificate path into noise. +const QUERY_ESCAPE: &percent_encoding::AsciiSet = &percent_encoding::NON_ALPHANUMERIC + .remove(b'-') + .remove(b'.') + .remove(b'_') + .remove(b'~') + .remove(b'/') + .remove(b':'); + +/// Build the PostgreSQL connection URL, including TLS settings. +/// +/// TLS parameters are only appended when configured, so leaving `ssl_mode` +/// unset keeps the driver's own resolution (and therefore `PGSSLMODE`) intact +/// for deployments set up before the setting existed. +fn postgres_url(config: &codex_config::PostgresConfig) -> String { + use percent_encoding::utf8_percent_encode as encode; + + let mut url = format!( + "postgres://{}:{}@{}:{}/{}", + encode(&config.username, USERINFO_ESCAPE), + encode(&config.password, USERINFO_ESCAPE), + config.host, + config.port, + config.database_name, + ); + + let mut params: Vec<(&str, String)> = Vec::new(); + if let Some(mode) = config.ssl_mode { + params.push(("sslmode", mode.as_str().to_string())); + } + if let Some(path) = &config.ssl_root_cert { + params.push(("sslrootcert", path.clone())); + } + if let Some(path) = &config.ssl_client_cert { + params.push(("sslcert", path.clone())); + } + if let Some(path) = &config.ssl_client_key { + params.push(("sslkey", path.clone())); + } + + if !params.is_empty() { + let query: Vec = params + .iter() + .map(|(k, v)| format!("{k}={}", encode(v, QUERY_ESCAPE))) + .collect(); + url.push('?'); + url.push_str(&query.join("&")); + } + url +} + impl Database { /// Validate pragma key to prevent SQL injection /// Only allows alphanumeric characters and underscores @@ -138,15 +202,7 @@ impl Database { .as_ref() .context("PostgreSQL configuration is required when db_type is postgres")?; - // Build connection string - let connection_string = format!( - "postgres://{}:{}@{}:{}/{}", - postgres_config.username, - postgres_config.password, - postgres_config.host, - postgres_config.port, - postgres_config.database_name - ); + let connection_string = postgres_url(postgres_config); // Configure connection pool options let mut opt = ConnectOptions::new(connection_string); @@ -515,6 +571,54 @@ impl Database { #[cfg(test)] mod tests { use super::*; + + use codex_config::{PgSslMode, PostgresConfig}; + + fn pg(f: impl FnOnce(&mut PostgresConfig)) -> String { + let mut config = PostgresConfig::default(); + f(&mut config); + postgres_url(&config) + } + + /// Interpolating the password directly meant one containing `@` or `/` + /// pointed the connection at a different host entirely. + #[test] + fn credentials_are_percent_encoded() { + let url = pg(|c| { + c.username = "co dex".to_string(); + c.password = "p@ss/w:rd?#".to_string(); + c.host = "db.internal".to_string(); + }); + + assert!( + url.starts_with("postgres://co%20dex:p%40ss%2Fw%3Ard%3F%23@db.internal:"), + "credentials must be escaped: {url}" + ); + } + + /// No TLS keys means no query string, so the driver keeps resolving + /// `PGSSLMODE` for deployments configured before the setting existed. + #[test] + fn no_tls_settings_leaves_the_url_untouched() { + let url = pg(|_| {}); + assert!(!url.contains('?'), "unexpected query string: {url}"); + } + + #[test] + fn the_tls_mode_reaches_the_url() { + let url = pg(|c| c.ssl_mode = Some(PgSslMode::VerifyFull)); + assert!(url.ends_with("?sslmode=verify-full"), "{url}"); + } + + #[test] + fn certificate_paths_reach_the_url() { + let url = pg(|c| { + c.ssl_mode = Some(PgSslMode::VerifyCa); + c.ssl_root_cert = Some("/etc/ssl/ca.crt".to_string()); + }); + assert!(url.contains("sslmode=verify-ca"), "{url}"); + assert!(url.contains("sslrootcert=/etc/ssl/ca.crt"), "{url}"); + } use codex_config::{DatabaseConfig, DatabaseType, SQLiteConfig}; use tempfile::TempDir; diff --git a/docs/docs/configuration.md b/docs/docs/configuration.md index 18da9d3fc..fb8162976 100644 --- a/docs/docs/configuration.md +++ b/docs/docs/configuration.md @@ -135,57 +135,47 @@ Overrunning the budget does not degrade gracefully: the server refuses new conne #### PostgreSQL TLS -TLS is **not** configured in `codex.yaml`. There is no `ssl_mode` setting. -Codex builds a plain `postgres://` URL and lets the driver apply the standard -libpq environment variables, so those are the supported way to control -transport security: +TLS is configured under `database.postgres`: -| Variable | Purpose | -|----------|---------| -| `PGSSLMODE` | Negotiation mode (see the table below). Defaults to `prefer`. | -| `PGSSLROOTCERT` | Path to the CA certificate used to verify the server. | -| `PGSSLCERT`, `PGSSLKEY` | Client certificate and key, for mutual TLS. | +```yaml +database: + postgres: + ssl_mode: verify-full + ssl_root_cert: /etc/ssl/certs/postgres-ca.crt + # ssl_client_cert / ssl_client_key for mutual TLS +``` + +| Setting | Purpose | +|---------|---------| +| `ssl_mode` | Negotiation mode (see the table below). Unset leaves the driver default, `prefer`. | +| `ssl_root_cert` | CA certificate used to verify the server. | +| `ssl_client_cert`, `ssl_client_key` | Client certificate and key, for mutual TLS. | -| `PGSSLMODE` | Encrypted | Certificate verified | Hostname verified | +| `ssl_mode` | Encrypted | Certificate verified | Hostname verified | |-------------|-----------|----------------------|-------------------| | `disable` | no | no | no | | `allow` | only if the server requires it | no | no | -| `prefer` *(default)* | only if the server offers it | **no** | **no** | +| `prefer` *(driver default)* | only if the server offers it | **no** | **no** | | `require` | yes | no | no | | `verify-ca` | yes | yes | no | | `verify-full` | yes | yes | yes | -:::warning The default guards against eavesdropping, not against interception -`prefer` encrypts the connection when the server offers TLS, but it accepts -**any** certificate and **silently falls back to an unencrypted connection** -when the server does not offer one. Neither the fallback nor a bogus +:::warning Leaving `ssl_mode` unset guards against eavesdropping, not interception +The driver default is `prefer`: it encrypts when the server offers TLS, but it +accepts **any** certificate and **silently falls back to an unencrypted +connection** when the server offers none. Neither the fallback nor a bogus certificate is logged, so a downgrade looks exactly like a healthy start. -That is adequate on a private network you control. It is not adequate over any -link an attacker could sit on. Require verification explicitly: - -```bash -PGSSLMODE=verify-full -PGSSLROOTCERT=/etc/ssl/certs/postgres-ca.crt -``` +That is adequate on a private network you control, and not much else. Set +`ssl_mode: verify-full` for anything remote or managed. ::: -:::note These are not `CODEX_` variables -They are read by the PostgreSQL driver rather than by Codex, so they carry no -`CODEX_` prefix and will not appear in `codex config check` output. They apply -to every command that opens a PostgreSQL connection, including `serve`, -`worker`, `migrate`, `export`, `import` and `copy`. -::: - -:::info Changing in Codex 2.0 -TLS becomes a first-class setting: `database.postgres.ssl_mode` and -`database.postgres.ssl_root_cert` in `codex.yaml`, with matching -`CODEX_DATABASE__POSTGRES__SSL_MODE` and `CODEX_DATABASE__POSTGRES__SSL_ROOT_CERT` -environment variables. `codex config check` will then validate them like any -other setting. - -The libpq variables above keep working as a fallback, so nothing you configure -now has to be undone. Configuring both will make the Codex setting win. +:::note The libpq variables still work +`PGSSLMODE`, `PGSSLROOTCERT`, `PGSSLCERT` and `PGSSLKEY` are read by the driver +when the corresponding Codex setting is unset, so a deployment configured that +way keeps working. The Codex setting wins when both are present. Being driver +variables they carry no `CODEX_` prefix and do not appear in `codex config +check`. ::: ## Application Configuration diff --git a/docs/docs/deployment/database.md b/docs/docs/deployment/database.md index 20b15d69c..d9f81972f 100644 --- a/docs/docs/deployment/database.md +++ b/docs/docs/deployment/database.md @@ -102,42 +102,40 @@ CODEX_DATABASE_POSTGRES_DATABASE_NAME=codex ### TLS -Codex has no TLS setting in `codex.yaml`. The PostgreSQL driver reads the -standard libpq environment variables and Codex does not override them, so set -them alongside the `CODEX_*` variables: +Configure TLS under `database.postgres`: -```bash -PGSSLMODE=verify-full -PGSSLROOTCERT=/etc/ssl/certs/postgres-ca.crt +```yaml +database: + postgres: + ssl_mode: verify-full + ssl_root_cert: /etc/ssl/certs/postgres-ca.crt ``` -Without `PGSSLMODE`, the driver uses `prefer`: it negotiates TLS when the -server offers it, accepts any certificate, and drops to an unencrypted -connection when the server offers nothing. Both the missing verification and -the downgrade are silent. Keep the default only when Codex and PostgreSQL sit -on a network you trust. +or with `CODEX_DATABASE__POSTGRES__SSL_MODE` and +`CODEX_DATABASE__POSTGRES__SSL_ROOT_CERT`. + +Leaving `ssl_mode` unset falls back to the driver's own resolution, which is +`prefer` unless `PGSSLMODE` says otherwise: it negotiates TLS when the server +offers it, accepts any certificate, and drops to an unencrypted connection when +the server offers nothing. Both the missing verification and the downgrade are +silent. Keep that only when Codex and PostgreSQL sit on a network you trust. Use `verify-full` for a managed or remote database. `verify-ca` skips the hostname check and is the fallback when the certificate's subject does not match the host you connect to. `require` encrypts without verifying anything, which stops passive capture but not an active attacker. -For mutual TLS, add `PGSSLCERT` and `PGSSLKEY`. +For mutual TLS, add `ssl_client_cert` and `ssl_client_key`. :::note -These are driver variables, so they have no `CODEX_` prefix and do not show up -in `codex config check`. They apply to every command that connects, including -`migrate`, `export`, `import` and `copy`. Note that a `?sslmode=` query -parameter on a `codex copy` URL is **not** honored: the URL is decomposed into -host, port, user, password and database name, and query parameters are -discarded. Use `PGSSLMODE` there too. -::: - -:::info Changing in Codex 2.0 -`ssl_mode` and `ssl_root_cert` become real settings under -`database.postgres`, validated by `codex config check` like anything else. The -libpq variables keep working as a fallback, so a `PGSSLMODE` you set now does -not need to be undone. See [Configuration](../configuration.md#postgresql-tls). +The libpq variables (`PGSSLMODE`, `PGSSLROOTCERT`, `PGSSLCERT`, `PGSSLKEY`) are +still read when the matching Codex setting is unset, so a deployment configured +that way keeps working; the Codex setting wins when both are present. + +A `?sslmode=` query parameter on a `codex copy` URL is **not** honored: the URL +is decomposed into host, port, user, password and database name, and query +parameters are discarded. Configure TLS on the config file for that side, or +use `PGSSLMODE`. ::: ### Connection Pooling From 4e29ad8ae84e02dc35ca6c83043c5dd59d719580 Mon Sep 17 00:00:00 2001 From: Sylvain Cau Date: Sat, 15 Aug 2026 18:17:53 -0700 Subject: [PATCH 08/11] docs(config)!: rewrite the configuration docs for the new environment scheme Adds an upgrade guide with the full rename table, generated from the configuration schema rather than written by hand so it cannot be partial. It covers the two settings whose sense inverted, the values that are now rejected instead of silently discarded, and the fact that startup no longer writes a config file. Sweeps every old variable name out of the documentation, the compose files, the Kubernetes manifests and the shipped example configs. The two inverted settings were excluded from the automated pass and rewritten by hand, since renaming `CODEX_DISABLE_WORKERS: "true"` to `CODEX_TASK__RUN_IN_PROCESS: "true"` would have silently reversed what those examples do. Verified by setting every variable the documentation mentions and running `codex config check`: nothing is flagged except the example `client_secret_env` target, which is correct, since that names an arbitrary variable rather than a setting. Two gaps surfaced while checking, both fixed: - A list field rejected an all-digit value, because figment parses one as a number before serde sees it, so `CODEX_API__CORS_ORIGINS=8080` failed. - `database.sqlite.pragmas` is a map the schema advertises as settable but had no lenient deserializer, so it could not actually be set from the environment. It now takes `k=v,k=v` like the other maps. --- config/config.docker.yaml | 16 +- config/config.kubernetes.yaml | 98 +++--- config/config.sqlite.yaml | 14 +- crates/codex-config/src/de.rs | 111 ++++-- crates/codex-config/src/types.rs | 1 + docker-compose.yml | 88 ++--- .../_partials/_docker-compose-quick-start.mdx | 2 +- .../_partials/_docker-run-quick-start.mdx | 2 +- docs/docs/api.md | 2 +- .../backup-migration/export-import-copy.md | 2 +- docs/docs/configuration.md | 162 ++++----- docs/docs/deployment/database.md | 12 +- docs/docs/deployment/docker.mdx | 64 ++-- docs/docs/deployment/kubernetes.md | 14 +- docs/docs/migration/v2-config.md | 317 ++++++++++++++++++ docs/docs/observability.md | 10 +- docs/docs/plugins/index.md | 2 +- docs/docs/third-party-apps.md | 6 +- docs/docs/troubleshooting.md | 4 +- docs/docs/users/oidc.md | 36 +- docs/sidebars.ts | 5 + 21 files changed, 679 insertions(+), 289 deletions(-) create mode 100644 docs/docs/migration/v2-config.md diff --git a/config/config.docker.yaml b/config/config.docker.yaml index e3fdcdc98..83f27937a 100644 --- a/config/config.docker.yaml +++ b/config/config.docker.yaml @@ -24,7 +24,7 @@ database: # Separate pool for in-process background work (task workers, scheduler, # pollers). NOTE: additive to max_connections; ensure the Postgres server's # max_connections covers the total when workers run in the same process. - # Multi-pod deployments run serve with CODEX_DISABLE_WORKERS=true, so no + # Multi-pod deployments run serve with task.run_in_process=false, so no # extra pool is created there. # background_max_connections: 16 # acquire_timeout_seconds: 30 @@ -44,7 +44,7 @@ logging: # Auth config - uses defaults, override with CODEX_AUTH_* env vars auth: - jwt_secret: "CHANGE_ME_IN_PRODUCTION" # Override with CODEX_AUTH_JWT_SECRET env var + jwt_secret: "CHANGE_ME_IN_PRODUCTION" # Override with CODEX_AUTH__JWT_SECRET env var # jwt_expiry_hours: 24 # refresh_token_enabled: true # refresh_token_expiry_days: 30 @@ -112,12 +112,12 @@ api: # ====================================================== # These settings control worker behavior and require a restart to take effect. task: - worker_count: 2 # Number of parallel task workers (override: CODEX_TASK_WORKER_COUNT) + worker_count: 2 # Number of parallel task workers (override: CODEX_TASK__WORKER_COUNT) # Scanner Settings (Startup-Time - Require Restart) # ================================================== scanner: - max_concurrent_scans: 2 # Max concurrent library scans (override: CODEX_SCANNER_MAX_CONCURRENT_SCANS) + max_concurrent_scans: 2 # Max concurrent library scans (override: CODEX_SCANNER__MAX_CONCURRENT_SCANS) # Scheduler Settings (Startup-Time - Require Restart) # ==================================================== @@ -125,16 +125,16 @@ scanner: # deduplication, thumbnail generation, etc.). # Note: The Docker TZ env var does NOT affect cron scheduling. Use this instead. # scheduler: -# timezone: UTC # IANA timezone (override: CODEX_SCHEDULER_TIMEZONE) +# timezone: UTC # IANA timezone (override: CODEX_SCHEDULER__TIMEZONE) # Files Settings (Startup-Time - Require Restart) # ================================================ # Directory paths for thumbnails, user uploads, and plugin data. # These default to {data_dir}/{subdir} and can be individually overridden. files: - thumbnail_dir: data/thumbnails # Thumbnail cache directory (override: CODEX_FILES_THUMBNAIL_DIR) - uploads_dir: data/uploads # User uploads directory (override: CODEX_FILES_UPLOADS_DIR) - plugins_dir: data/plugins # Plugin file storage directory (override: CODEX_FILES_PLUGINS_DIR) + thumbnail_dir: data/thumbnails # Thumbnail cache directory (override: CODEX_FILES__THUMBNAIL_DIR) + uploads_dir: data/uploads # User uploads directory (override: CODEX_FILES__UPLOADS_DIR) + plugins_dir: data/plugins # Plugin file storage directory (override: CODEX_FILES__PLUGINS_DIR) # PDF Rendering Settings # ====================== diff --git a/config/config.kubernetes.yaml b/config/config.kubernetes.yaml index b9c0564b0..dbddcc63f 100644 --- a/config/config.kubernetes.yaml +++ b/config/config.kubernetes.yaml @@ -4,67 +4,67 @@ # In Kubernetes, set these via Secrets/ConfigMaps: # # Required Environment Variables: -# CODEX_DATABASE_DB_TYPE=postgres -# CODEX_DATABASE_POSTGRES_HOST=postgres-service -# CODEX_DATABASE_POSTGRES_PORT=5432 -# CODEX_DATABASE_POSTGRES_USERNAME= -# CODEX_DATABASE_POSTGRES_PASSWORD= -# CODEX_DATABASE_POSTGRES_DATABASE_NAME=codex -# CODEX_AUTH_JWT_SECRET= +# CODEX_DATABASE__DB_TYPE=postgres +# CODEX_DATABASE__POSTGRES__HOST=postgres-service +# CODEX_DATABASE__POSTGRES__PORT=5432 +# CODEX_DATABASE__POSTGRES__USERNAME= +# CODEX_DATABASE__POSTGRES__PASSWORD= +# CODEX_DATABASE__POSTGRES__DATABASE_NAME=codex +# CODEX_AUTH__JWT_SECRET= # # Optional overrides: -# CODEX_APPLICATION_HOST=0.0.0.0 -# CODEX_APPLICATION_PORT=8080 -# CODEX_API_ENABLE_API_DOCS=false -# CODEX_LOGGING_LEVEL=info -# CODEX_LOGGING_CONSOLE=true -# CODEX_LOGGING_FILE=/var/log/codex/codex.log +# CODEX_APPLICATION__HOST=0.0.0.0 +# CODEX_APPLICATION__PORT=8080 +# CODEX_API__ENABLE_API_DOCS=false +# CODEX_LOGGING__LEVEL=info +# CODEX_LOGGING__CONSOLE=true +# CODEX_LOGGING__FILE=/var/log/codex/codex.log # # Startup-Time Settings (Require Pod Restart) # ============================================ # These settings are in the config file and require a pod restart to change. # Override via environment variables if needed: -# CODEX_TASK_WORKER_COUNT=2 -# CODEX_SCANNER_MAX_CONCURRENT_SCANS=2 -# CODEX_SCHEDULER_TIMEZONE=UTC # IANA timezone for cron jobs -# CODEX_FILES_THUMBNAIL_DIR=data/thumbnails -# CODEX_FILES_UPLOADS_DIR=data/uploads +# CODEX_TASK__WORKER_COUNT=2 +# CODEX_SCANNER__MAX_CONCURRENT_SCANS=2 +# CODEX_SCHEDULER__TIMEZONE=UTC # IANA timezone for cron jobs +# CODEX_FILES__THUMBNAIL_DIR=data/thumbnails +# CODEX_FILES__UPLOADS_DIR=data/uploads # # PDF Settings: -# CODEX_PDF_PDFIUM_LIBRARY_PATH=/usr/lib/libpdfium.so -# CODEX_PDF_RENDER_DPI=150 -# CODEX_PDF_JPEG_QUALITY=85 -# CODEX_PDF_CACHE_RENDERED_PAGES=true -# CODEX_PDF_CACHE_DIR=data/cache +# CODEX_PDF__PDFIUM_LIBRARY_PATH=/usr/lib/libpdfium.so +# CODEX_PDF__RENDER_DPI=150 +# CODEX_PDF__JPEG_QUALITY=85 +# CODEX_PDF__CACHE_RENDERED_PAGES=true +# CODEX_PDF__CACHE_DIR=data/cache # # Komga API (for third-party app compatibility): -# CODEX_KOMGA_API_ENABLED=false -# CODEX_KOMGA_API_PREFIX=komga +# CODEX_KOMGA_API__ENABLED=false +# CODEX_KOMGA_API__PREFIX=komga # # OIDC / SSO Authentication: -# CODEX_AUTH_OIDC_ENABLED=true -# CODEX_AUTH_OIDC_AUTO_CREATE_USERS=true -# CODEX_AUTH_OIDC_DEFAULT_ROLE=reader # admin, maintainer, reader -# CODEX_AUTH_OIDC_REDIRECT_URI_BASE=https://codex.example.com # Optional, defaults to http://{application.host}:{application.port} +# CODEX_AUTH__OIDC__ENABLED=true +# CODEX_AUTH__OIDC__AUTO_CREATE_USERS=true +# CODEX_AUTH__OIDC__DEFAULT_ROLE=reader # admin, maintainer, reader +# CODEX_AUTH__OIDC__REDIRECT_URI_BASE=https://codex.example.com # Optional, defaults to http://{application.host}:{application.port} # # Per-provider settings (replace AUTHENTIK with your provider name): -# CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_DISPLAY_NAME=Authentik SSO -# CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_ISSUER_URL=https://authentik.example.com/application/o/codex/ -# CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_CLIENT_ID= -# CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_CLIENT_SECRET= -# CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_SCOPES=profile,email -# CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_GROUPS_CLAIM=groups -# CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_USERNAME_CLAIM=preferred_username -# CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_EMAIL_CLAIM=email +# CODEX_AUTH__OIDC__PROVIDERS__AUTHENTIK__DISPLAY_NAME=Authentik SSO +# CODEX_AUTH__OIDC__PROVIDERS__AUTHENTIK__ISSUER_URL=https://authentik.example.com/application/o/codex/ +# CODEX_AUTH__OIDC__PROVIDERS__AUTHENTIK__CLIENT_ID= +# CODEX_AUTH__OIDC__PROVIDERS__AUTHENTIK__CLIENT_SECRET= +# CODEX_AUTH__OIDC__PROVIDERS__AUTHENTIK__SCOPES=profile,email +# CODEX_AUTH__OIDC__PROVIDERS__AUTHENTIK__GROUPS_CLAIM=groups +# CODEX_AUTH__OIDC__PROVIDERS__AUTHENTIK__USERNAME_CLAIM=preferred_username +# CODEX_AUTH__OIDC__PROVIDERS__AUTHENTIK__EMAIL_CLAIM=email # # Rate Limiting: -# CODEX_RATE_LIMIT_ENABLED=true -# CODEX_RATE_LIMIT_ANONYMOUS_RPS=10 -# CODEX_RATE_LIMIT_ANONYMOUS_BURST=50 -# CODEX_RATE_LIMIT_AUTHENTICATED_RPS=50 -# CODEX_RATE_LIMIT_AUTHENTICATED_BURST=200 -# CODEX_RATE_LIMIT_EXEMPT_PATHS=/health,/api/v1/events -# CODEX_RATE_LIMIT_CLEANUP_INTERVAL_SECS=60 -# CODEX_RATE_LIMIT_BUCKET_TTL_SECS=300 +# CODEX_RATE_LIMIT__ENABLED=true +# CODEX_RATE_LIMIT__ANONYMOUS_RPS=10 +# CODEX_RATE_LIMIT__ANONYMOUS_BURST=50 +# CODEX_RATE_LIMIT__AUTHENTICATED_RPS=50 +# CODEX_RATE_LIMIT__AUTHENTICATED_BURST=200 +# CODEX_RATE_LIMIT__EXEMPT_PATHS=/health,/api/v1/events +# CODEX_RATE_LIMIT__CLEANUP_INTERVAL_SECS=60 +# CODEX_RATE_LIMIT__BUCKET_TTL_SECS=300 task: worker_count: 2 # Number of parallel task workers per pod @@ -75,7 +75,7 @@ scanner: # Scheduler Settings (Startup-Time - Require Pod Restart) # ======================================================= # scheduler: -# timezone: UTC # IANA timezone for cron jobs (override: CODEX_SCHEDULER_TIMEZONE) +# timezone: UTC # IANA timezone for cron jobs (override: CODEX_SCHEDULER__TIMEZONE) files: thumbnail_dir: data/thumbnails # Thumbnail cache directory (persistent volume mount) @@ -172,9 +172,9 @@ files: # # Most fields can also be set via env (CODEX_OBSERVABILITY_*) so secrets # (auth tokens) can come from Kubernetes Secrets: -# CODEX_OBSERVABILITY_ENABLED=true -# CODEX_OBSERVABILITY_OTLP_ENDPOINT=http://otel-collector.observability:4317 -# CODEX_OBSERVABILITY_OTLP_HEADERS=signoz-access-token=$(cat /secrets/signoz-token) +# CODEX_OBSERVABILITY__ENABLED=true +# CODEX_OBSERVABILITY__OTLP__ENDPOINT=http://otel-collector.observability:4317 +# CODEX_OBSERVABILITY__OTLP__HEADERS=signoz-access-token=$(cat /secrets/signoz-token) # observability: # enabled: true # service_name: codex diff --git a/config/config.sqlite.yaml b/config/config.sqlite.yaml index bae7b7318..c083ea7b3 100644 --- a/config/config.sqlite.yaml +++ b/config/config.sqlite.yaml @@ -42,7 +42,7 @@ logging: # Auth config - uses defaults, override with CODEX_AUTH_* env vars # Defaults: JWT 24h expiry, Argon2 (19456, 2, 1) auth: - jwt_secret: "CHANGE_ME_IN_PRODUCTION" # Override with CODEX_AUTH_JWT_SECRET env var + jwt_secret: "CHANGE_ME_IN_PRODUCTION" # Override with CODEX_AUTH__JWT_SECRET env var # jwt_expiry_hours: 24 # refresh_token_enabled: true # refresh_token_expiry_days: 30 @@ -112,28 +112,28 @@ auth: # ====================================================== # These settings control worker behavior and require a restart to take effect. task: - worker_count: 2 # Number of parallel task workers (override: CODEX_TASK_WORKER_COUNT) + worker_count: 2 # Number of parallel task workers (override: CODEX_TASK__WORKER_COUNT) # Scanner Settings (Startup-Time - Require Restart) # ================================================== scanner: - max_concurrent_scans: 2 # Max concurrent library scans (override: CODEX_SCANNER_MAX_CONCURRENT_SCANS) + max_concurrent_scans: 2 # Max concurrent library scans (override: CODEX_SCANNER__MAX_CONCURRENT_SCANS) # Scheduler Settings (Startup-Time - Require Restart) # ==================================================== # Controls the timezone for all cron-based scheduled tasks (library scans, # deduplication, thumbnail generation, etc.). # scheduler: -# timezone: UTC # IANA timezone (override: CODEX_SCHEDULER_TIMEZONE) +# timezone: UTC # IANA timezone (override: CODEX_SCHEDULER__TIMEZONE) # Files Settings (Startup-Time - Require Restart) # ================================================ # Directory paths for thumbnails, user uploads, and plugin data. # These default to {data_dir}/{subdir} and can be individually overridden. files: - thumbnail_dir: data/thumbnails # Thumbnail cache directory (override: CODEX_FILES_THUMBNAIL_DIR) - uploads_dir: data/uploads # User uploads directory (override: CODEX_FILES_UPLOADS_DIR) - plugins_dir: data/plugins # Plugin file storage directory (override: CODEX_FILES_PLUGINS_DIR) + thumbnail_dir: data/thumbnails # Thumbnail cache directory (override: CODEX_FILES__THUMBNAIL_DIR) + uploads_dir: data/uploads # User uploads directory (override: CODEX_FILES__UPLOADS_DIR) + plugins_dir: data/plugins # Plugin file storage directory (override: CODEX_FILES__PLUGINS_DIR) # PDF Rendering Settings # ====================== diff --git a/crates/codex-config/src/de.rs b/crates/codex-config/src/de.rs index 725b9d099..ce69d7c61 100644 --- a/crates/codex-config/src/de.rs +++ b/crates/codex-config/src/de.rs @@ -138,6 +138,16 @@ where Ok(split_csv(value)) } + // figment turns an all-digit value into a number before serde sees it, + // so a single numeric entry would otherwise fail to deserialize. + fn visit_u64(self, value: u64) -> Result, E> { + Ok(vec![value.to_string()]) + } + + fn visit_i64(self, value: i64) -> Result, E> { + Ok(vec![value.to_string()]) + } + fn visit_seq>(self, mut seq: A) -> Result, A::Error> { let mut out = Vec::new(); while let Some(item) = seq.next_element::()? { @@ -158,40 +168,87 @@ pub fn string_map<'de, D>(deserializer: D) -> Result, D: where D: Deserializer<'de>, { - struct StringMap; + deserializer.deserialize_any(StringMapVisitor) +} + +struct StringMapVisitor; + +impl<'de> Visitor<'de> for StringMapVisitor { + type Value = HashMap; + + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("a map, or a comma-separated list of key=value pairs") + } + + fn visit_str(self, value: &str) -> Result { + let mut out = HashMap::new(); + for entry in split_csv(value) { + let (key, val) = entry + .split_once('=') + .ok_or_else(|| E::custom(format!("expected `key=value`, found `{entry}`")))?; + let key = key.trim(); + if key.is_empty() { + return Err(E::custom(format!("empty key in `{entry}`"))); + } + out.insert(key.to_string(), val.trim().to_string()); + } + Ok(out) + } + + fn visit_map>(self, mut map: A) -> Result { + let mut out = HashMap::new(); + while let Some((key, value)) = map.next_entry::()? { + out.insert(key, value); + } + Ok(out) + } +} + +/// An optional string map, empty meaning absent. +/// +/// `database.sqlite.pragmas` is `Option>`, so it needs the option +/// wrapper for the same reason `optional_truthy_bool` does: serde hands the +/// inner value to the option's deserializer, bypassing [`string_map`]. +pub fn optional_string_map<'de, D>( + deserializer: D, +) -> Result>, D::Error> +where + D: Deserializer<'de>, +{ + struct OptionalStringMap; - impl<'de> Visitor<'de> for StringMap { - type Value = HashMap; + impl<'de> Visitor<'de> for OptionalStringMap { + type Value = Option>; fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.write_str("a map, or a comma-separated list of key=value pairs") + f.write_str("a map, `key=value` pairs, or nothing") + } + + fn visit_none(self) -> Result { + Ok(None) + } + + fn visit_unit(self) -> Result { + Ok(None) } fn visit_str(self, value: &str) -> Result { - let mut out = HashMap::new(); - for entry in split_csv(value) { - let (key, val) = entry - .split_once('=') - .ok_or_else(|| E::custom(format!("expected `key=value`, found `{entry}`")))?; - let key = key.trim(); - if key.is_empty() { - return Err(E::custom(format!("empty key in `{entry}`"))); - } - out.insert(key.to_string(), val.trim().to_string()); + if value.is_empty() { + return Ok(None); } - Ok(out) + StringMapVisitor.visit_str(value).map(Some) } - fn visit_map>(self, mut map: A) -> Result { - let mut out = HashMap::new(); - while let Some((key, value)) = map.next_entry::()? { - out.insert(key, value); - } - Ok(out) + fn visit_map>(self, map: A) -> Result { + StringMapVisitor.visit_map(map).map(Some) + } + + fn visit_some>(self, inner: D2) -> Result { + inner.deserialize_any(StringMapVisitor).map(Some) } } - deserializer.deserialize_any(StringMap) + deserializer.deserialize_option(OptionalStringMap) } /// A map whose values are themselves comma-separated lists. @@ -379,6 +436,16 @@ mod tests { assert!(empty.is_empty()); } + /// figment parses an all-digit value as a number, so a one-element list + /// of digits must still work. + #[test] + fn lists_accept_a_bare_number() { + let parsed = serde_yaml::from_str::("value: 8080") + .unwrap() + .value; + assert_eq!(parsed, vec!["8080"]); + } + #[test] fn maps_accept_a_mapping_or_key_value_pairs() { let from_map = serde_yaml::from_str::("value:\n a: '1'\n b: '2'\n") diff --git a/crates/codex-config/src/types.rs b/crates/codex-config/src/types.rs index dd3a5184c..a1322e306 100644 --- a/crates/codex-config/src/types.rs +++ b/crates/codex-config/src/types.rs @@ -865,6 +865,7 @@ impl Default for PostgresConfig { #[serde(default)] pub struct SQLiteConfig { pub path: String, + #[serde(deserialize_with = "crate::de::optional_string_map")] pub pragmas: Option>, // Connection Pool Settings diff --git a/docker-compose.yml b/docker-compose.yml index 747876b33..37db50a1e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -50,21 +50,21 @@ services: # Configuration overrides (optional - uses CODEX_ prefix) # These override values in the config file # Application configuration - # CODEX_APPLICATION_HOST: 0.0.0.0 - # CODEX_APPLICATION_PORT: 8080 + # CODEX_APPLICATION__HOST: 0.0.0.0 + # CODEX_APPLICATION__PORT: 8080 # Database connection - # CODEX_DATABASE_DB_TYPE: postgres - # CODEX_DATABASE_POSTGRES_HOST: postgres - # CODEX_DATABASE_POSTGRES_PORT: 5432 - # CODEX_DATABASE_POSTGRES_USERNAME: codex - # CODEX_DATABASE_POSTGRES_PASSWORD: codex - # CODEX_DATABASE_POSTGRES_DATABASE_NAME: codex + # CODEX_DATABASE__DB_TYPE: postgres + # CODEX_DATABASE__POSTGRES__HOST: postgres + # CODEX_DATABASE__POSTGRES__PORT: 5432 + # CODEX_DATABASE__POSTGRES__USERNAME: codex + # CODEX_DATABASE__POSTGRES__PASSWORD: codex + # CODEX_DATABASE__POSTGRES__DATABASE_NAME: codex # # Startup-time settings (in config file, require restart): - # CODEX_TASK_WORKER_COUNT: 4 # Number of parallel task workers per pod - # CODEX_SCANNER_MAX_CONCURRENT_SCANS: 2 # Maximum number of concurrent library scans - # CODEX_FILES_THUMBNAIL_DIR: data/thumbnails # Thumbnail cache directory (default: data/thumbnails) + # CODEX_TASK__WORKER_COUNT: 4 # Number of parallel task workers per pod + # CODEX_SCANNER__MAX_CONCURRENT_SCANS: 2 # Maximum number of concurrent library scans + # CODEX_FILES__THUMBNAIL_DIR: data/thumbnails # Thumbnail cache directory (default: data/thumbnails) networks: - codex-network restart: unless-stopped @@ -114,34 +114,34 @@ services: # Encryption key for plugin secrets (base64-encoded 32-byte key) # Generate with: openssl rand -base64 32 CODEX_ENCRYPTION_KEY: "pjImnrzPzSmuvBKkzWAlTzrfyZ9O3pU/9IKuRT94Y/w=" - # Disable workers in web container (workers run in separate container) - CODEX_DISABLE_WORKERS: "true" + # Workers run in a separate container + CODEX_TASK__RUN_IN_PROCESS: "false" # OAuth redirect URI base (must match the externally-facing URL users see in their browser) # In dev, the Vite frontend on :5173 proxies API calls to the backend - CODEX_AUTH_OIDC_REDIRECT_URI_BASE: "http://localhost:5173" + CODEX_AUTH__OIDC__REDIRECT_URI_BASE: "http://localhost:5173" # Configuration overrides (optional - uses CODEX_ prefix) # Uncomment and modify as needed to override config.docker.yaml values # Database connection - # CODEX_DATABASE_DB_TYPE: postgres - # CODEX_DATABASE_POSTGRES_HOST: postgres - # CODEX_DATABASE_POSTGRES_PORT: 5432 - # CODEX_DATABASE_POSTGRES_USERNAME: codex - # CODEX_DATABASE_POSTGRES_PASSWORD: codex - # CODEX_DATABASE_POSTGRES_DATABASE_NAME: codex - CODEX_SCHEDULER_TIMEZONE: America/Los_Angeles - CODEX_LOGGING_LEVEL: debug + # CODEX_DATABASE__DB_TYPE: postgres + # CODEX_DATABASE__POSTGRES__HOST: postgres + # CODEX_DATABASE__POSTGRES__PORT: 5432 + # CODEX_DATABASE__POSTGRES__USERNAME: codex + # CODEX_DATABASE__POSTGRES__PASSWORD: codex + # CODEX_DATABASE__POSTGRES__DATABASE_NAME: codex + CODEX_SCHEDULER__TIMEZONE: America/Los_Angeles + CODEX_LOGGING__LEVEL: debug # OpenTelemetry observability: ship traces/metrics to the bundled Jaeger # sidecar so `make dev-up` "just works". The Codex config files keep # observability disabled by default (trust posture for production # deployments); the dev compose overrides that here. - CODEX_OBSERVABILITY_ENABLED: "true" - CODEX_OBSERVABILITY_SERVICE_NAME: codex - CODEX_OBSERVABILITY_OTLP_ENDPOINT: http://jaeger:4317 - CODEX_OBSERVABILITY_OTLP_PROTOCOL: grpc + CODEX_OBSERVABILITY__ENABLED: "true" + CODEX_OBSERVABILITY__SERVICE_NAME: codex + CODEX_OBSERVABILITY__OTLP__ENDPOINT: http://jaeger:4317 + CODEX_OBSERVABILITY__OTLP__PROTOCOL: grpc # The browser RUM proxy always speaks OTLP/HTTP; point it at Jaeger's # HTTP port (4318) instead of reusing the gRPC endpoint above. - CODEX_OBSERVABILITY_OTLP_PROXY_ENDPOINT: http://jaeger:4318 - CODEX_OBSERVABILITY_BROWSER_ENABLED: "true" + CODEX_OBSERVABILITY__OTLP__PROXY_ENDPOINT: http://jaeger:4318 + CODEX_OBSERVABILITY__BROWSER__ENABLED: "true" healthcheck: test: ["CMD-SHELL", "curl -f http://localhost:8080/health || exit 1"] interval: 10s @@ -159,7 +159,7 @@ services: - dev # Codex worker (dedicated worker container for development) - # Note: Workers also check migrations on startup, but will skip if CODEX_SKIP_MIGRATIONS is set + # Note: workers check migrations on startup; database.run_migrations=false makes them wait instead codex-dev-worker: build: context: . @@ -226,22 +226,22 @@ services: # Encryption key for plugin secrets (must match codex-dev) CODEX_ENCRYPTION_KEY: "pjImnrzPzSmuvBKkzWAlTzrfyZ9O3pU/9IKuRT94Y/w=" # Worker count (can also be set in config file) - CODEX_TASK_WORKER_COUNT: "2" - CODEX_SKIP_MIGRATIONS: "true" + CODEX_TASK__WORKER_COUNT: "2" + CODEX_DATABASE__RUN_MIGRATIONS: "false" # Database connection - # CODEX_DATABASE_DB_TYPE: postgres - # CODEX_DATABASE_POSTGRES_HOST: postgres - # CODEX_DATABASE_POSTGRES_PORT: 5432 - # CODEX_DATABASE_POSTGRES_USERNAME: codex - # CODEX_DATABASE_POSTGRES_PASSWORD: codex - # CODEX_DATABASE_POSTGRES_DATABASE_NAME: codex - CODEX_LOGGING_LEVEL: debug + # CODEX_DATABASE__DB_TYPE: postgres + # CODEX_DATABASE__POSTGRES__HOST: postgres + # CODEX_DATABASE__POSTGRES__PORT: 5432 + # CODEX_DATABASE__POSTGRES__USERNAME: codex + # CODEX_DATABASE__POSTGRES__PASSWORD: codex + # CODEX_DATABASE__POSTGRES__DATABASE_NAME: codex + CODEX_LOGGING__LEVEL: debug # OpenTelemetry observability: same overrides as codex-dev so the worker # emits spans/metrics into the same Jaeger sidecar. - CODEX_OBSERVABILITY_ENABLED: "true" - CODEX_OBSERVABILITY_SERVICE_NAME: codex - CODEX_OBSERVABILITY_OTLP_ENDPOINT: http://jaeger:4317 - CODEX_OBSERVABILITY_OTLP_PROTOCOL: grpc + CODEX_OBSERVABILITY__ENABLED: "true" + CODEX_OBSERVABILITY__SERVICE_NAME: codex + CODEX_OBSERVABILITY__OTLP__ENDPOINT: http://jaeger:4317 + CODEX_OBSERVABILITY__OTLP__PROTOCOL: grpc networks: - codex-network profiles: @@ -373,7 +373,7 @@ services: # Accepts OTLP natively on 4317 (gRPC) / 4318 (HTTP), serves the UI on 16686, # and stores spans in memory. Available in the dev profile; the codex-dev and # codex-dev-worker services above are pre-wired to send OTLP here via - # CODEX_OBSERVABILITY_OTLP_ENDPOINT=http://jaeger:4317. + # CODEX_OBSERVABILITY__OTLP__ENDPOINT=http://jaeger:4317. jaeger: image: jaegertracing/all-in-one:1.62.0 container_name: codex-jaeger @@ -449,7 +449,7 @@ services: command: ["codex", "serve", "--config", "/app/config/config.screenshots.yaml"] environment: - CODEX_LOGGING_LEVEL: info + CODEX_LOGGING__LEVEL: info CODEX_ENCRYPTION_KEY: "pjImnrzPzSmuvBKkzWAlTzrfyZ9O3pU/9IKuRT94Y/w=" healthcheck: test: ["CMD-SHELL", "curl -f http://localhost:8080/health || exit 1"] diff --git a/docs/docs/_partials/_docker-compose-quick-start.mdx b/docs/docs/_partials/_docker-compose-quick-start.mdx index 881c93a0b..637232ade 100644 --- a/docs/docs/_partials/_docker-compose-quick-start.mdx +++ b/docs/docs/_partials/_docker-compose-quick-start.mdx @@ -15,7 +15,7 @@ services: PUID: 1000 PGID: 1000 # Generate with: openssl rand -base64 32 - CODEX_AUTH_JWT_SECRET: "your-secure-secret-here" + CODEX_AUTH__JWT_SECRET: "your-secure-secret-here" restart: unless-stopped volumes: diff --git a/docs/docs/_partials/_docker-run-quick-start.mdx b/docs/docs/_partials/_docker-run-quick-start.mdx index 04f93bd2c..b889ff1c4 100644 --- a/docs/docs/_partials/_docker-run-quick-start.mdx +++ b/docs/docs/_partials/_docker-run-quick-start.mdx @@ -6,7 +6,7 @@ docker run -d \ -v codex-data:/app/data \ -e PUID=1000 \ -e PGID=1000 \ - -e CODEX_AUTH_JWT_SECRET="$(openssl rand -base64 32)" \ + -e CODEX_AUTH__JWT_SECRET="$(openssl rand -base64 32)" \ ghcr.io/ashdevfr/codex:latest ``` diff --git a/docs/docs/api.md b/docs/docs/api.md index 8eaba190c..81b3d4181 100644 --- a/docs/docs/api.md +++ b/docs/docs/api.md @@ -30,7 +30,7 @@ api: Or via environment variable: ```bash -CODEX_API_ENABLE_API_DOCS=true +CODEX_API__ENABLE_API_DOCS=true ``` ### Accessing API Docs diff --git a/docs/docs/backup-migration/export-import-copy.md b/docs/docs/backup-migration/export-import-copy.md index 6f1d120d2..098c53c9d 100644 --- a/docs/docs/backup-migration/export-import-copy.md +++ b/docs/docs/backup-migration/export-import-copy.md @@ -136,7 +136,7 @@ What you need to prepare depends on the engine: - **SQLite target — nothing to create.** Like `serve`, `import` writes a default config if none exists and creates the database file (and its parent directories) automatically. Just point `database.sqlite.path` (or - `CODEX_DATABASE_SQLITE_PATH`) at the destination and run it. + `CODEX_DATABASE__SQLITE__PATH`) at the destination and run it. - **PostgreSQL target — create the empty database and role first.** PostgreSQL won't create a database from a connection string, so provision it once (your Kubernetes chart / operator / an init job typically does this): diff --git a/docs/docs/configuration.md b/docs/docs/configuration.md index fb8162976..ff2c1024e 100644 --- a/docs/docs/configuration.md +++ b/docs/docs/configuration.md @@ -128,7 +128,7 @@ These pool sizes are **per process**, while PostgreSQL's own `max_connections` ( (web replicas x max_connections) + (workers x max_connections) + jobs <= server max_connections - superuser_reserved_connections ``` -`background_max_connections` is **additive** to `max_connections` whenever task workers run in the same process as the web server. Multi-pod deployments run the web server with `CODEX_DISABLE_WORKERS=true`, so no background pool is created there. +`background_max_connections` is **additive** to `max_connections` whenever task workers run in the same process as the web server. Multi-pod deployments run the web server with `CODEX_TASK__RUN_IN_PROCESS=false`, so no background pool is created there. Overrunning the budget does not degrade gracefully: the server refuses new connections with `FATAL: remaining connection slots are reserved`, and because a starting process needs a connection before it can do anything, **new pods fail to start while the running ones carry on looking healthy**. Codex logs a warning at startup when its pools do not fit, including when the connections already open on the server leave no room. ::: @@ -350,7 +350,7 @@ scheduler: | Setting | Default | Env Override | Description | |---------|---------|--------------|-------------| -| `timezone` | `UTC` | `CODEX_SCHEDULER_TIMEZONE` | Default IANA timezone for all cron schedules | +| `timezone` | `UTC` | `CODEX_SCHEDULER__TIMEZONE` | Default IANA timezone for all cron schedules | The timezone must be a valid IANA timezone name (e.g., `America/New_York`, `Europe/London`, `Asia/Tokyo`). Abbreviations like `PST` or offsets like `UTC+8` are **not** supported. @@ -361,7 +361,7 @@ Priority: **Library `cronTimezone`** > **Server `scheduler.timezone`** > **UTC** ::: :::note Docker Users -The Docker `TZ` environment variable does **not** affect the cron scheduler. You must set `CODEX_SCHEDULER_TIMEZONE` (or configure `scheduler.timezone` in your YAML) for cron jobs to run in your local timezone. +The Docker `TZ` environment variable does **not** affect the cron scheduler. You must set `CODEX_SCHEDULER__TIMEZONE` (or configure `scheduler.timezone` in your YAML) for cron jobs to run in your local timezone. ::: ## Files Configuration @@ -395,7 +395,7 @@ plugin SDK applies it to its own logger and exposes it so each plugin can adopt it for its own logging. Plugins honor it on a best-effort basis. :::tip -Set `logging.level: debug` (or `CODEX_LOGGING_LEVEL=debug`) when debugging a +Set `logging.level: debug` (or `CODEX_LOGGING__LEVEL=debug`) when debugging a misbehaving plugin to surface its diagnostic logging, then revert to `info` to keep logs quiet. Note that this makes the host verbose too. ::: @@ -480,7 +480,7 @@ wget -O- https://github.com/bblanchon/pdfium-binaries/releases/latest/download/p 1. Download `pdfium-win-x64.zip` from [bblanchon/pdfium-binaries releases](https://github.com/bblanchon/pdfium-binaries/releases) 2. Extract `pdfium.dll` to a directory in your `PATH` -3. Or set `CODEX_PDF_PDFIUM_LIBRARY_PATH` to the full path of `pdfium.dll` +3. Or set `CODEX_PDF__PDFIUM_LIBRARY_PATH` to the full path of `pdfium.dll` ### Without PDFium @@ -675,7 +675,7 @@ rate_limit: Or via environment variable: ```bash -CODEX_RATE_LIMIT_ENABLED=false +CODEX_RATE_LIMIT__ENABLED=false ``` :::caution @@ -782,17 +782,17 @@ observability: | Setting | Default | Env Override | Description | |---------|---------|--------------|-------------| -| `enabled` | `false` | `CODEX_OBSERVABILITY_ENABLED` | Master switch. No providers are initialized when `false`. | -| `service_name` | `codex` | `CODEX_OBSERVABILITY_SERVICE_NAME` | Resource attribute that identifies this process in the backend UI. | +| `enabled` | `false` | `CODEX_OBSERVABILITY__ENABLED` | Master switch. No providers are initialized when `false`. | +| `service_name` | `codex` | `CODEX_OBSERVABILITY__SERVICE_NAME` | Resource attribute that identifies this process in the backend UI. | ### OTLP exporter (`observability.otlp`) | Setting | Default | Env Override | Description | |---------|---------|--------------|-------------| -| `endpoint` | `""` | `CODEX_OBSERVABILITY_OTLP_ENDPOINT` | Collector URL. Required when `enabled: true`. | -| `protocol` | `grpc` | `CODEX_OBSERVABILITY_OTLP_PROTOCOL` | One of `grpc`, `http/protobuf`, `http/json`. | -| `headers` | `{}` | `CODEX_OBSERVABILITY_OTLP_HEADERS` | Map of arbitrary headers. Env format: `k1=v1,k2=v2`. | -| `timeout_ms` | `5000` | `CODEX_OBSERVABILITY_OTLP_TIMEOUT_MS` | Per-export request timeout. | +| `endpoint` | `""` | `CODEX_OBSERVABILITY__OTLP__ENDPOINT` | Collector URL. Required when `enabled: true`. | +| `protocol` | `grpc` | `CODEX_OBSERVABILITY__OTLP__PROTOCOL` | One of `grpc`, `http/protobuf`, `http/json`. | +| `headers` | `{}` | `CODEX_OBSERVABILITY__OTLP__HEADERS` | Map of arbitrary headers. Env format: `k1=v1,k2=v2`. | +| `timeout_ms` | `5000` | `CODEX_OBSERVABILITY__OTLP__TIMEOUT_MS` | Per-export request timeout. | :::tip Endpoint format For gRPC endpoints, include the scheme: `http://host:4317` (cleartext) or `https://host:4317` (TLS). @@ -803,8 +803,8 @@ For HTTP endpoints, point at the base URL only: `http://collector:4318`. The SDK | Setting | Default | Env Override | Description | |---------|---------|--------------|-------------| -| `enabled` | `true` | `CODEX_OBSERVABILITY_TRACES_ENABLED` | Per-signal switch. Honored only when the parent `enabled` is also true. | -| `sample_ratio` | `1.0` | `CODEX_OBSERVABILITY_TRACES_SAMPLE_RATIO` | Parent-based sampler ratio in `[0.0, 1.0]`. Out-of-range values are clamped. | +| `enabled` | `true` | `CODEX_OBSERVABILITY__TRACES__ENABLED` | Per-signal switch. Honored only when the parent `enabled` is also true. | +| `sample_ratio` | `1.0` | `CODEX_OBSERVABILITY__TRACES__SAMPLE_RATIO` | Parent-based sampler ratio in `[0.0, 1.0]`. Out-of-range values are clamped. | See the [sampling guidance table](./observability#sampling-guidance) for production-sized recommendations. @@ -812,16 +812,16 @@ See the [sampling guidance table](./observability#sampling-guidance) for product | Setting | Default | Env Override | Description | |---------|---------|--------------|-------------| -| `enabled` | `true` | `CODEX_OBSERVABILITY_METRICS_ENABLED` | Per-signal switch. Honored only when the parent `enabled` is also true. | -| `export_interval_ms` | `30000` | `CODEX_OBSERVABILITY_METRICS_EXPORT_INTERVAL_MS` | Periodic reader export interval. Lower values increase load on the collector. | +| `enabled` | `true` | `CODEX_OBSERVABILITY__METRICS__ENABLED` | Per-signal switch. Honored only when the parent `enabled` is also true. | +| `export_interval_ms` | `30000` | `CODEX_OBSERVABILITY__METRICS__EXPORT_INTERVAL_MS` | Periodic reader export interval. Lower values increase load on the collector. | ### Browser RUM (`observability.browser`) | Setting | Default | Env Override | Description | |---------|---------|--------------|-------------| -| `enabled` | `false` | `CODEX_OBSERVABILITY_BROWSER_ENABLED` | Opt-in switch for the OTLP proxy and the SPA's SDK bootstrap. | -| `proxy_path` | `/api/v1/observability/otlp` | `CODEX_OBSERVABILITY_BROWSER_PROXY_PATH` | Path on the Codex server where the browser SDK POSTs OTLP batches. | -| `sample_ratio` | `0.1` | `CODEX_OBSERVABILITY_BROWSER_SAMPLE_RATIO` | Client-side sample ratio. | +| `enabled` | `false` | `CODEX_OBSERVABILITY__BROWSER__ENABLED` | Opt-in switch for the OTLP proxy and the SPA's SDK bootstrap. | +| `proxy_path` | `/api/v1/observability/otlp` | `CODEX_OBSERVABILITY__BROWSER__PROXY_PATH` | Path on the Codex server where the browser SDK POSTs OTLP batches. | +| `sample_ratio` | `0.1` | `CODEX_OBSERVABILITY__BROWSER__SAMPLE_RATIO` | Client-side sample ratio. | :::note Two independent switches `observability.browser.enabled` is intentionally independent from the backend `observability.enabled` flag. Some operators want server-side observability without shipping spans from every browser tab. The SDK additionally refuses to start if `observability.otlp.endpoint` is empty, so a misconfigured server cannot leak data via the browser. @@ -840,7 +840,7 @@ Configuration paths are converted to environment variables: :::warning These names change in Codex 2.0 Because a single `_` separates both nesting levels and the words inside a field -name, `CODEX_RATE_LIMIT_ANONYMOUS_RPS` is ambiguous: nothing in the name says +name, `CODEX_RATE_LIMIT__ANONYMOUS_RPS` is ambiguous: nothing in the name says the section is `rate_limit` rather than `rate`. Codex 2.0 uses `__` between nesting levels instead, so that variable becomes `CODEX_RATE_LIMIT__ANONYMOUS_RPS`. @@ -852,94 +852,94 @@ the new spelling, and a variable renamed early is silently ignored. | Config Path | Environment Variable | |-------------|---------------------| -| `database.db_type` | `CODEX_DATABASE_DB_TYPE` | -| `database.postgres.host` | `CODEX_DATABASE_POSTGRES_HOST` | -| `auth.jwt_secret` | `CODEX_AUTH_JWT_SECRET` | -| `logging.level` | `CODEX_LOGGING_LEVEL` | -| `scheduler.timezone` | `CODEX_SCHEDULER_TIMEZONE` | +| `database.db_type` | `CODEX_DATABASE__DB_TYPE` | +| `database.postgres.host` | `CODEX_DATABASE__POSTGRES__HOST` | +| `auth.jwt_secret` | `CODEX_AUTH__JWT_SECRET` | +| `logging.level` | `CODEX_LOGGING__LEVEL` | +| `scheduler.timezone` | `CODEX_SCHEDULER__TIMEZONE` | ### Common Environment Variables ```bash # Database -CODEX_DATABASE_DB_TYPE=postgres -CODEX_DATABASE_POSTGRES_HOST=localhost -CODEX_DATABASE_POSTGRES_PORT=5432 -CODEX_DATABASE_POSTGRES_USERNAME=codex -CODEX_DATABASE_POSTGRES_PASSWORD=secret -CODEX_DATABASE_POSTGRES_DATABASE_NAME=codex +CODEX_DATABASE__DB_TYPE=postgres +CODEX_DATABASE__POSTGRES__HOST=localhost +CODEX_DATABASE__POSTGRES__PORT=5432 +CODEX_DATABASE__POSTGRES__USERNAME=codex +CODEX_DATABASE__POSTGRES__PASSWORD=secret +CODEX_DATABASE__POSTGRES__DATABASE_NAME=codex # Application -CODEX_APPLICATION_HOST=0.0.0.0 -CODEX_APPLICATION_PORT=8080 +CODEX_APPLICATION__HOST=0.0.0.0 +CODEX_APPLICATION__PORT=8080 # Authentication -CODEX_AUTH_JWT_SECRET=your-secure-secret-key +CODEX_AUTH__JWT_SECRET=your-secure-secret-key # Logging -CODEX_LOGGING_LEVEL=debug -CODEX_LOGGING_FILE=/var/log/codex/codex.log +CODEX_LOGGING__LEVEL=debug +CODEX_LOGGING__FILE=/var/log/codex/codex.log # API -CODEX_API_ENABLE_API_DOCS=true +CODEX_API__ENABLE_API_DOCS=true # Task Workers -CODEX_TASK_WORKER_COUNT=4 +CODEX_TASK__WORKER_COUNT=4 # Scanner -CODEX_SCANNER_MAX_CONCURRENT_SCANS=2 +CODEX_SCANNER__MAX_CONCURRENT_SCANS=2 # Scheduler -CODEX_SCHEDULER_TIMEZONE=America/Los_Angeles +CODEX_SCHEDULER__TIMEZONE=America/Los_Angeles # Files (thumbnails and uploads) -CODEX_FILES_THUMBNAIL_DIR=data/thumbnails -CODEX_FILES_UPLOADS_DIR=data/uploads +CODEX_FILES__THUMBNAIL_DIR=data/thumbnails +CODEX_FILES__UPLOADS_DIR=data/uploads # PDF Rendering -# CODEX_PDF_PDFIUM_LIBRARY_PATH=/usr/local/lib/libpdfium.so # Optional, auto-detected -CODEX_PDF_RENDER_DPI=150 -CODEX_PDF_JPEG_QUALITY=85 -CODEX_PDF_CACHE_RENDERED_PAGES=true -CODEX_PDF_CACHE_DIR=data/cache +# CODEX_PDF__PDFIUM_LIBRARY_PATH=/usr/local/lib/libpdfium.so # Optional, auto-detected +CODEX_PDF__RENDER_DPI=150 +CODEX_PDF__JPEG_QUALITY=85 +CODEX_PDF__CACHE_RENDERED_PAGES=true +CODEX_PDF__CACHE_DIR=data/cache # PDF Handle Cache (in-memory open-document cache) -CODEX_PDF_HANDLE_CACHE_ENABLED=true -CODEX_PDF_HANDLE_CACHE_CAPACITY=256 -CODEX_PDF_HANDLE_CACHE_IDLE_TTL_MINUTES=15 -CODEX_PDF_HANDLE_CACHE_SWEEP_INTERVAL_SECONDS=60 +CODEX_PDF_HANDLE_CACHE__ENABLED=true +CODEX_PDF_HANDLE_CACHE__CAPACITY=256 +CODEX_PDF_HANDLE_CACHE__IDLE_TTL_MINUTES=15 +CODEX_PDF_HANDLE_CACHE__SWEEP_INTERVAL_SECONDS=60 # Komga-Compatible API -CODEX_KOMGA_API_ENABLED=true -CODEX_KOMGA_API_PREFIX=komga +CODEX_KOMGA_API__ENABLED=true +CODEX_KOMGA_API__PREFIX=komga # Plugin Credential Encryption CODEX_ENCRYPTION_KEY=your-base64-encoded-32-byte-key # Rate Limiting -CODEX_RATE_LIMIT_ENABLED=true -CODEX_RATE_LIMIT_ANONYMOUS_RPS=10 -CODEX_RATE_LIMIT_ANONYMOUS_BURST=50 -CODEX_RATE_LIMIT_AUTHENTICATED_RPS=50 -CODEX_RATE_LIMIT_AUTHENTICATED_BURST=200 -CODEX_RATE_LIMIT_EXEMPT_PATHS=/health,/api/v1/events -CODEX_RATE_LIMIT_CLEANUP_INTERVAL_SECS=60 -CODEX_RATE_LIMIT_BUCKET_TTL_SECS=300 +CODEX_RATE_LIMIT__ENABLED=true +CODEX_RATE_LIMIT__ANONYMOUS_RPS=10 +CODEX_RATE_LIMIT__ANONYMOUS_BURST=50 +CODEX_RATE_LIMIT__AUTHENTICATED_RPS=50 +CODEX_RATE_LIMIT__AUTHENTICATED_BURST=200 +CODEX_RATE_LIMIT__EXEMPT_PATHS=/health,/api/v1/events +CODEX_RATE_LIMIT__CLEANUP_INTERVAL_SECS=60 +CODEX_RATE_LIMIT__BUCKET_TTL_SECS=300 # Observability (OpenTelemetry / OTLP) -CODEX_OBSERVABILITY_ENABLED=true -CODEX_OBSERVABILITY_SERVICE_NAME=codex -CODEX_OBSERVABILITY_OTLP_ENDPOINT=http://localhost:4317 -CODEX_OBSERVABILITY_OTLP_PROTOCOL=grpc -CODEX_OBSERVABILITY_OTLP_HEADERS=signoz-access-token=abc123,x-tenant=production -CODEX_OBSERVABILITY_OTLP_TIMEOUT_MS=5000 -CODEX_OBSERVABILITY_TRACES_ENABLED=true -CODEX_OBSERVABILITY_TRACES_SAMPLE_RATIO=0.1 -CODEX_OBSERVABILITY_METRICS_ENABLED=true -CODEX_OBSERVABILITY_METRICS_EXPORT_INTERVAL_MS=30000 -CODEX_OBSERVABILITY_BROWSER_ENABLED=false -CODEX_OBSERVABILITY_BROWSER_PROXY_PATH=/api/v1/observability/otlp -CODEX_OBSERVABILITY_BROWSER_SAMPLE_RATIO=0.1 +CODEX_OBSERVABILITY__ENABLED=true +CODEX_OBSERVABILITY__SERVICE_NAME=codex +CODEX_OBSERVABILITY__OTLP__ENDPOINT=http://localhost:4317 +CODEX_OBSERVABILITY__OTLP__PROTOCOL=grpc +CODEX_OBSERVABILITY__OTLP__HEADERS=signoz-access-token=abc123,x-tenant=production +CODEX_OBSERVABILITY__OTLP__TIMEOUT_MS=5000 +CODEX_OBSERVABILITY__TRACES__ENABLED=true +CODEX_OBSERVABILITY__TRACES__SAMPLE_RATIO=0.1 +CODEX_OBSERVABILITY__METRICS__ENABLED=true +CODEX_OBSERVABILITY__METRICS__EXPORT_INTERVAL_MS=30000 +CODEX_OBSERVABILITY__BROWSER__ENABLED=false +CODEX_OBSERVABILITY__BROWSER__PROXY_PATH=/api/v1/observability/otlp +CODEX_OBSERVABILITY__BROWSER__SAMPLE_RATIO=0.1 ``` ## Runtime vs Startup Settings @@ -1060,13 +1060,13 @@ files: Set these via Kubernetes ConfigMaps and Secrets: ```bash -CODEX_DATABASE_DB_TYPE=postgres -CODEX_DATABASE_POSTGRES_HOST=postgres-service -CODEX_DATABASE_POSTGRES_PORT=5432 -CODEX_DATABASE_POSTGRES_USERNAME= -CODEX_DATABASE_POSTGRES_PASSWORD= -CODEX_DATABASE_POSTGRES_DATABASE_NAME=codex -CODEX_AUTH_JWT_SECRET= +CODEX_DATABASE__DB_TYPE=postgres +CODEX_DATABASE__POSTGRES__HOST=postgres-service +CODEX_DATABASE__POSTGRES__PORT=5432 +CODEX_DATABASE__POSTGRES__USERNAME= +CODEX_DATABASE__POSTGRES__PASSWORD= +CODEX_DATABASE__POSTGRES__DATABASE_NAME=codex +CODEX_AUTH__JWT_SECRET= CODEX_ENCRYPTION_KEY= ``` diff --git a/docs/docs/deployment/database.md b/docs/docs/deployment/database.md index d9f81972f..61909facc 100644 --- a/docs/docs/deployment/database.md +++ b/docs/docs/deployment/database.md @@ -92,12 +92,12 @@ database: Or via environment variables: ```bash -CODEX_DATABASE_DB_TYPE=postgres -CODEX_DATABASE_POSTGRES_HOST=localhost -CODEX_DATABASE_POSTGRES_PORT=5432 -CODEX_DATABASE_POSTGRES_USERNAME=codex -CODEX_DATABASE_POSTGRES_PASSWORD=your-secure-password -CODEX_DATABASE_POSTGRES_DATABASE_NAME=codex +CODEX_DATABASE__DB_TYPE=postgres +CODEX_DATABASE__POSTGRES__HOST=localhost +CODEX_DATABASE__POSTGRES__PORT=5432 +CODEX_DATABASE__POSTGRES__USERNAME=codex +CODEX_DATABASE__POSTGRES__PASSWORD=your-secure-password +CODEX_DATABASE__POSTGRES__DATABASE_NAME=codex ``` ### TLS diff --git a/docs/docs/deployment/docker.mdx b/docs/docs/deployment/docker.mdx index 54d17fdb9..2dcb3da8c 100644 --- a/docs/docs/deployment/docker.mdx +++ b/docs/docs/deployment/docker.mdx @@ -53,7 +53,7 @@ services: # Codex data - codex-data:/app/data environment: - CODEX_AUTH_JWT_SECRET: "your-secure-secret-here" + CODEX_AUTH__JWT_SECRET: "your-secure-secret-here" restart: unless-stopped volumes: @@ -78,7 +78,7 @@ services: # Use a local directory for data - ./codex-data:/app/data environment: - CODEX_AUTH_JWT_SECRET: "your-secure-secret-here" + CODEX_AUTH__JWT_SECRET: "your-secure-secret-here" restart: unless-stopped ``` @@ -118,14 +118,14 @@ services: - /path/to/your/library:/library:ro - codex-data:/app/data environment: - CODEX_AUTH_JWT_SECRET: "your-secure-secret-here" - CODEX_DATABASE_DB_TYPE: postgres - CODEX_DATABASE_POSTGRES_HOST: postgres - CODEX_DATABASE_POSTGRES_PORT: 5432 - CODEX_DATABASE_POSTGRES_USERNAME: codex - CODEX_DATABASE_POSTGRES_PASSWORD: codex - CODEX_DATABASE_POSTGRES_DATABASE_NAME: codex - # CODEX_SCHEDULER_TIMEZONE: America/Los_Angeles # IANA timezone for cron jobs (default: UTC) + CODEX_AUTH__JWT_SECRET: "your-secure-secret-here" + CODEX_DATABASE__DB_TYPE: postgres + CODEX_DATABASE__POSTGRES__HOST: postgres + CODEX_DATABASE__POSTGRES__PORT: 5432 + CODEX_DATABASE__POSTGRES__USERNAME: codex + CODEX_DATABASE__POSTGRES__PASSWORD: codex + CODEX_DATABASE__POSTGRES__DATABASE_NAME: codex + # CODEX_SCHEDULER__TIMEZONE: America/Los_Angeles # IANA timezone for cron jobs (default: UTC) restart: unless-stopped volumes: @@ -134,7 +134,7 @@ volumes: ``` :::tip Timezone for Cron Jobs -The Docker `TZ` environment variable does **not** affect Codex cron scheduling. To run scheduled library scans in your local timezone, set `CODEX_SCHEDULER_TIMEZONE` to a valid [IANA timezone name](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) (e.g., `America/Los_Angeles`, `Europe/London`). Each library can also override this with its own timezone in the scanning configuration. +The Docker `TZ` environment variable does **not** affect Codex cron scheduling. To run scheduled library scans in your local timezone, set `CODEX_SCHEDULER__TIMEZONE` to a valid [IANA timezone name](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) (e.g., `America/Los_Angeles`, `Europe/London`). Each library can also override this with its own timezone in the scanning configuration. ::: ## Separate Worker Containers (Advanced) @@ -158,13 +158,13 @@ x-codex-common: &codex-common - /path/to/your/library:/library:ro - codex-data:/app/data environment: &codex-env - CODEX_AUTH_JWT_SECRET: "your-secure-secret-here" - CODEX_DATABASE_DB_TYPE: postgres - CODEX_DATABASE_POSTGRES_HOST: postgres - CODEX_DATABASE_POSTGRES_PORT: 5432 - CODEX_DATABASE_POSTGRES_USERNAME: codex - CODEX_DATABASE_POSTGRES_PASSWORD: codex - CODEX_DATABASE_POSTGRES_DATABASE_NAME: codex + CODEX_AUTH__JWT_SECRET: "your-secure-secret-here" + CODEX_DATABASE__DB_TYPE: postgres + CODEX_DATABASE__POSTGRES__HOST: postgres + CODEX_DATABASE__POSTGRES__PORT: 5432 + CODEX_DATABASE__POSTGRES__USERNAME: codex + CODEX_DATABASE__POSTGRES__PASSWORD: codex + CODEX_DATABASE__POSTGRES__DATABASE_NAME: codex depends_on: postgres: condition: service_healthy @@ -195,8 +195,8 @@ services: - "8080:8080" environment: <<: *codex-env - # Disable workers in the web container - CODEX_DISABLE_WORKERS: "true" + # Workers run in their own container + CODEX_TASK__RUN_IN_PROCESS: "false" healthcheck: test: ["CMD-SHELL", "curl -f http://localhost:8080/health || exit 1"] interval: 10s @@ -212,9 +212,9 @@ services: environment: <<: *codex-env # Number of parallel task workers - CODEX_TASK_WORKER_COUNT: "4" - # Skip migrations (web container handles them) - CODEX_SKIP_MIGRATIONS: "true" + CODEX_TASK__WORKER_COUNT: "4" + # The web container applies migrations; this one waits for them + CODEX_DATABASE__RUN_MIGRATIONS: "false" depends_on: postgres: condition: service_healthy @@ -230,9 +230,9 @@ volumes: | Variable | Description | Default | |----------|-------------|---------| -| `CODEX_DISABLE_WORKERS` | Disable background workers in web container | `false` | -| `CODEX_SKIP_MIGRATIONS` | Skip database migrations on startup | `false` | -| `CODEX_TASK_WORKER_COUNT` | Number of parallel task workers | `4` | +| `CODEX_TASK__RUN_IN_PROCESS` | Run background workers in this container | `true` | +| `CODEX_DATABASE__RUN_MIGRATIONS` | Apply migrations at startup; when false, wait for them | `true` | +| `CODEX_TASK__WORKER_COUNT` | Number of parallel task workers | `4` | ### Scaling Workers @@ -290,7 +290,7 @@ services: environment: PUID: 1000 PGID: 1000 - CODEX_AUTH_JWT_SECRET: "your-secret-here" + CODEX_AUTH__JWT_SECRET: "your-secret-here" volumes: - ./codex-data:/app/data - ./codex-config:/app/config @@ -309,11 +309,11 @@ Common environment variables for Docker: |----------|-------------|---------| | `PUID` | User ID for file permissions | `1000` | | `PGID` | Group ID for file permissions | `1000` | -| `CODEX_AUTH_JWT_SECRET` | JWT signing secret | `your-secret-key` | -| `CODEX_DATABASE_DB_TYPE` | Database type | `postgres` or `sqlite` | -| `CODEX_DATABASE_POSTGRES_HOST` | PostgreSQL host | `postgres` | -| `CODEX_DATABASE_POSTGRES_PASSWORD` | PostgreSQL password | `secret` | -| `CODEX_LOGGING_LEVEL` | Log level | `info`, `debug` | +| `CODEX_AUTH__JWT_SECRET` | JWT signing secret | `your-secret-key` | +| `CODEX_DATABASE__DB_TYPE` | Database type | `postgres` or `sqlite` | +| `CODEX_DATABASE__POSTGRES__HOST` | PostgreSQL host | `postgres` | +| `CODEX_DATABASE__POSTGRES__PASSWORD` | PostgreSQL password | `secret` | +| `CODEX_LOGGING__LEVEL` | Log level | `info`, `debug` | See [Configuration](../configuration) for all options. diff --git a/docs/docs/deployment/kubernetes.md b/docs/docs/deployment/kubernetes.md index 3432fcbe5..8eb8830c1 100644 --- a/docs/docs/deployment/kubernetes.md +++ b/docs/docs/deployment/kubernetes.md @@ -67,14 +67,14 @@ spec: image: codex:latest args: ["config", "check", "--strict", "--quiet"] env: - - name: CODEX_DATABASE_DB_TYPE + - name: CODEX_DATABASE__DB_TYPE value: "postgres" - - name: CODEX_DATABASE_POSTGRES_HOST + - name: CODEX_DATABASE__POSTGRES__HOST valueFrom: configMapKeyRef: name: codex-config key: postgres-host - - name: CODEX_AUTH_JWT_SECRET + - name: CODEX_AUTH__JWT_SECRET valueFrom: secretKeyRef: name: codex-secrets @@ -85,19 +85,19 @@ spec: ports: - containerPort: 8080 env: - - name: CODEX_DATABASE_DB_TYPE + - name: CODEX_DATABASE__DB_TYPE value: "postgres" - - name: CODEX_DATABASE_POSTGRES_HOST + - name: CODEX_DATABASE__POSTGRES__HOST valueFrom: configMapKeyRef: name: codex-config key: postgres-host - - name: CODEX_DATABASE_POSTGRES_PASSWORD + - name: CODEX_DATABASE__POSTGRES__PASSWORD valueFrom: secretKeyRef: name: codex-secrets key: postgres-password - - name: CODEX_AUTH_JWT_SECRET + - name: CODEX_AUTH__JWT_SECRET valueFrom: secretKeyRef: name: codex-secrets diff --git a/docs/docs/migration/v2-config.md b/docs/docs/migration/v2-config.md new file mode 100644 index 000000000..7ae6a0836 --- /dev/null +++ b/docs/docs/migration/v2-config.md @@ -0,0 +1,317 @@ +--- +sidebar_position: 1 +--- + +# Upgrading to Codex 2.0 + +Codex 2.0 renames every configuration environment variable. Nothing else about +your setup has to change: config files keep the same shape, and there is no +database migration beyond the usual automatic one. + +**Codex will not start with the old names.** That is deliberate. The +alternative was to ignore them, which means a server running with default rate +limits, the wrong port, or workers in a pod meant to serve web traffic, and no +indication anything is wrong. A refusal at startup, listing exactly what to +change, is the safer failure. + +## Before you upgrade + +On 1.44 or later, ask the running version what will break: + +```bash +codex config check +``` + +It prints every variable you set that changes, with its replacement. Prepare +the edit, then apply it at the same time you bump the image tag. + +:::warning Do not rename before upgrading +1.x does not read the new spelling. A variable renamed early is silently +ignored, which is the failure mode this release exists to remove. +::: + +## What changed, and why + +The old scheme used a single `_` for two different jobs: separating nesting +levels, and separating words inside a key. Nothing in +`CODEX_RATE_LIMIT_ANONYMOUS_RPS` says whether the section is `rate_limit` or +`rate`, so every variable needed a hand-written rule, and adding a setting +meant remembering to add one. Several documented variables never had one and +did nothing at all. + +Codex 2.0 separates nesting levels with `__` and keeps `_` for words within a +key: + +``` +rate_limit.anonymous_rps -> CODEX_RATE_LIMIT__ANONYMOUS_RPS +database.postgres.max_connections -> CODEX_DATABASE__POSTGRES__MAX_CONNECTIONS +pdf_handle_cache.capacity -> CODEX_PDF_HANDLE_CACHE__CAPACITY +``` + +## Settings that moved + +These are not simple renames. Two invert their meaning, so read them carefully: + +| Before | After | Note | +| ------ | ----- | ---- | +| `CODEX_COOKIE_SECURE` | `CODEX_AUTH__COOKIE_SECURE` | same meaning | +| `CODEX_DISABLE_WORKERS` | `CODEX_TASK__RUN_IN_PROCESS` | **inverted**: `DISABLE_WORKERS=true` becomes `RUN_IN_PROCESS=false` | +| `CODEX_IMAGE_DECODE_CONCURRENCY` | `CODEX_IMAGES__DECODE_CONCURRENCY` | same meaning | +| `CODEX_MIGRATION_WAIT_INTERVAL` | `CODEX_DATABASE__MIGRATION_WAIT_INTERVAL_SECS` | same meaning | +| `CODEX_MIGRATION_WAIT_TIMEOUT` | `CODEX_DATABASE__MIGRATION_WAIT_TIMEOUT_SECS` | same meaning | +| `CODEX_PLUGIN_ALLOWED_COMMANDS` | `CODEX_PLUGINS__ALLOWED_COMMANDS` | same meaning | +| `CODEX_SKIP_MIGRATIONS` | `CODEX_DATABASE__RUN_MIGRATIONS` | **inverted**: `SKIP_MIGRATIONS=true` becomes `RUN_MIGRATIONS=false` | + +All seven are now ordinary config keys, so they can live in `codex.yaml` +instead of the environment. + +## Settings that stay environment-only + +`CODEX_ENCRYPTION_KEY`, `CODEX_SOURCE_DATABASE_URL` and +`CODEX_TARGET_DATABASE_URL` are unchanged. + +## Other behaviour changes + +**A bad value now stops the server.** The old override layer discarded values +it could not parse, so `CODEX_KOMGA_API_ENABLED=ture` quietly meant `false`. +Unrecognized input is now an error naming the variable. + +**Startup no longer writes a config file.** Codex used to serialize its +defaults to disk when the file was missing, which produced an uncommented dump +and, because those defaults were read from the environment, could capture a +database password in plaintext. Run `codex config init` for a commented +starter instead. + +**Values keep the shapes you already use.** Booleans still accept +`true`/`false`, `1`/`0`, `yes`/`no` and `on`/`off`. Lists still accept a +comma-separated string. `CODEX_OBSERVABILITY__OTLP__HEADERS` still accepts +`k1=v1,k2=v2`. An empty value still means "unset". + +## New in this release + +**Local overlay.** A `codex.local.yaml` beside your config is merged on top of +it, so secrets and per-host tweaks no longer mean editing the committed file. +It merges key by key; a list in the overlay replaces the base list rather than +extending it. + +**TOML.** Config files may be `.toml` as well as `.yaml`, chosen by extension. + +**PostgreSQL TLS.** `database.postgres.ssl_mode` and friends are real settings. +See [Configuration](../configuration.md#postgresql-tls). The libpq variables +(`PGSSLMODE` and so on) still work when the Codex setting is unset. + +## Full rename table + +Generated from the configuration schema, so it is exhaustive. + +### `api` + +| Before | After | +| ------ | ----- | +| `CODEX_API_API_DOCS_PATH` | `CODEX_API__API_DOCS_PATH` | +| `CODEX_API_BASE_PATH` | `CODEX_API__BASE_PATH` | +| `CODEX_API_CORS_ENABLED` | `CODEX_API__CORS_ENABLED` | +| `CODEX_API_CORS_ORIGINS` | `CODEX_API__CORS_ORIGINS` | +| `CODEX_API_ENABLE_API_DOCS` | `CODEX_API__ENABLE_API_DOCS` | +| `CODEX_API_MAX_PAGE_SIZE` | `CODEX_API__MAX_PAGE_SIZE` | + +### `application` + +| Before | After | +| ------ | ----- | +| `CODEX_APPLICATION_BASE_URL` | `CODEX_APPLICATION__BASE_URL` | +| `CODEX_APPLICATION_HOST` | `CODEX_APPLICATION__HOST` | +| `CODEX_APPLICATION_PORT` | `CODEX_APPLICATION__PORT` | + +### `auth` + +| Before | After | +| ------ | ----- | +| `CODEX_AUTH_ARGON2_MEMORY_COST` | `CODEX_AUTH__ARGON2_MEMORY_COST` | +| `CODEX_AUTH_ARGON2_PARALLELISM` | `CODEX_AUTH__ARGON2_PARALLELISM` | +| `CODEX_AUTH_ARGON2_TIME_COST` | `CODEX_AUTH__ARGON2_TIME_COST` | +| `CODEX_AUTH_COOKIE_SECURE` | `CODEX_AUTH__COOKIE_SECURE` | +| `CODEX_AUTH_EMAIL_CONFIRMATION_REQUIRED` | `CODEX_AUTH__EMAIL_CONFIRMATION_REQUIRED` | +| `CODEX_AUTH_JWT_EXPIRY_HOURS` | `CODEX_AUTH__JWT_EXPIRY_HOURS` | +| `CODEX_AUTH_JWT_SECRET` | `CODEX_AUTH__JWT_SECRET` | +| `CODEX_AUTH_OIDC_ALLOWED_REDIRECT_URIS` | `CODEX_AUTH__OIDC__ALLOWED_REDIRECT_URIS` | +| `CODEX_AUTH_OIDC_AUTO_CREATE_USERS` | `CODEX_AUTH__OIDC__AUTO_CREATE_USERS` | +| `CODEX_AUTH_OIDC_DEFAULT_ROLE` | `CODEX_AUTH__OIDC__DEFAULT_ROLE` | +| `CODEX_AUTH_OIDC_ENABLED` | `CODEX_AUTH__OIDC__ENABLED` | +| `CODEX_AUTH_OIDC_PROVIDERS` | `CODEX_AUTH__OIDC__PROVIDERS` | +| `CODEX_AUTH_OIDC_PROVIDERS_*_ROLE_MAPPING` | `CODEX_AUTH__OIDC__PROVIDERS__*__ROLE_MAPPING` | +| `CODEX_AUTH_OIDC_REDIRECT_URI_BASE` | `CODEX_AUTH__OIDC__REDIRECT_URI_BASE` | +| `CODEX_AUTH_REFRESH_TOKEN_ENABLED` | `CODEX_AUTH__REFRESH_TOKEN_ENABLED` | +| `CODEX_AUTH_REFRESH_TOKEN_EXPIRY_DAYS` | `CODEX_AUTH__REFRESH_TOKEN_EXPIRY_DAYS` | + +### `database` + +| Before | After | +| ------ | ----- | +| `CODEX_DATABASE_DB_TYPE` | `CODEX_DATABASE__DB_TYPE` | +| `CODEX_DATABASE_MIGRATION_WAIT_INTERVAL_SECS` | `CODEX_DATABASE__MIGRATION_WAIT_INTERVAL_SECS` | +| `CODEX_DATABASE_MIGRATION_WAIT_TIMEOUT_SECS` | `CODEX_DATABASE__MIGRATION_WAIT_TIMEOUT_SECS` | +| `CODEX_DATABASE_POSTGRES_ACQUIRE_TIMEOUT_SECONDS` | `CODEX_DATABASE__POSTGRES__ACQUIRE_TIMEOUT_SECONDS` | +| `CODEX_DATABASE_POSTGRES_BACKGROUND_MAX_CONNECTIONS` | `CODEX_DATABASE__POSTGRES__BACKGROUND_MAX_CONNECTIONS` | +| `CODEX_DATABASE_POSTGRES_BATCH_FAN_OUT` | `CODEX_DATABASE__POSTGRES__BATCH_FAN_OUT` | +| `CODEX_DATABASE_POSTGRES_DATABASE_NAME` | `CODEX_DATABASE__POSTGRES__DATABASE_NAME` | +| `CODEX_DATABASE_POSTGRES_HOST` | `CODEX_DATABASE__POSTGRES__HOST` | +| `CODEX_DATABASE_POSTGRES_IDLE_TIMEOUT_SECONDS` | `CODEX_DATABASE__POSTGRES__IDLE_TIMEOUT_SECONDS` | +| `CODEX_DATABASE_POSTGRES_MAX_CONNECTIONS` | `CODEX_DATABASE__POSTGRES__MAX_CONNECTIONS` | +| `CODEX_DATABASE_POSTGRES_MAX_LIFETIME_SECONDS` | `CODEX_DATABASE__POSTGRES__MAX_LIFETIME_SECONDS` | +| `CODEX_DATABASE_POSTGRES_MIN_CONNECTIONS` | `CODEX_DATABASE__POSTGRES__MIN_CONNECTIONS` | +| `CODEX_DATABASE_POSTGRES_OPERATION_DEADLINE_SECONDS` | `CODEX_DATABASE__POSTGRES__OPERATION_DEADLINE_SECONDS` | +| `CODEX_DATABASE_POSTGRES_PASSWORD` | `CODEX_DATABASE__POSTGRES__PASSWORD` | +| `CODEX_DATABASE_POSTGRES_PORT` | `CODEX_DATABASE__POSTGRES__PORT` | +| `CODEX_DATABASE_POSTGRES_SSL_CLIENT_CERT` | `CODEX_DATABASE__POSTGRES__SSL_CLIENT_CERT` | +| `CODEX_DATABASE_POSTGRES_SSL_CLIENT_KEY` | `CODEX_DATABASE__POSTGRES__SSL_CLIENT_KEY` | +| `CODEX_DATABASE_POSTGRES_SSL_MODE` | `CODEX_DATABASE__POSTGRES__SSL_MODE` | +| `CODEX_DATABASE_POSTGRES_SSL_ROOT_CERT` | `CODEX_DATABASE__POSTGRES__SSL_ROOT_CERT` | +| `CODEX_DATABASE_POSTGRES_USERNAME` | `CODEX_DATABASE__POSTGRES__USERNAME` | +| `CODEX_DATABASE_RUN_MIGRATIONS` | `CODEX_DATABASE__RUN_MIGRATIONS` | +| `CODEX_DATABASE_SQLITE_ACQUIRE_TIMEOUT_SECONDS` | `CODEX_DATABASE__SQLITE__ACQUIRE_TIMEOUT_SECONDS` | +| `CODEX_DATABASE_SQLITE_BACKGROUND_MAX_CONNECTIONS` | `CODEX_DATABASE__SQLITE__BACKGROUND_MAX_CONNECTIONS` | +| `CODEX_DATABASE_SQLITE_BATCH_FAN_OUT` | `CODEX_DATABASE__SQLITE__BATCH_FAN_OUT` | +| `CODEX_DATABASE_SQLITE_IDLE_TIMEOUT_SECONDS` | `CODEX_DATABASE__SQLITE__IDLE_TIMEOUT_SECONDS` | +| `CODEX_DATABASE_SQLITE_MAX_CONNECTIONS` | `CODEX_DATABASE__SQLITE__MAX_CONNECTIONS` | +| `CODEX_DATABASE_SQLITE_MAX_LIFETIME_SECONDS` | `CODEX_DATABASE__SQLITE__MAX_LIFETIME_SECONDS` | +| `CODEX_DATABASE_SQLITE_MIN_CONNECTIONS` | `CODEX_DATABASE__SQLITE__MIN_CONNECTIONS` | +| `CODEX_DATABASE_SQLITE_OPERATION_DEADLINE_SECONDS` | `CODEX_DATABASE__SQLITE__OPERATION_DEADLINE_SECONDS` | +| `CODEX_DATABASE_SQLITE_PATH` | `CODEX_DATABASE__SQLITE__PATH` | +| `CODEX_DATABASE_SQLITE_PRAGMAS` | `CODEX_DATABASE__SQLITE__PRAGMAS` | + +### `email` + +| Before | After | +| ------ | ----- | +| `CODEX_EMAIL_SMTP_FROM_EMAIL` | `CODEX_EMAIL__SMTP_FROM_EMAIL` | +| `CODEX_EMAIL_SMTP_FROM_NAME` | `CODEX_EMAIL__SMTP_FROM_NAME` | +| `CODEX_EMAIL_SMTP_HOST` | `CODEX_EMAIL__SMTP_HOST` | +| `CODEX_EMAIL_SMTP_PASSWORD` | `CODEX_EMAIL__SMTP_PASSWORD` | +| `CODEX_EMAIL_SMTP_PORT` | `CODEX_EMAIL__SMTP_PORT` | +| `CODEX_EMAIL_SMTP_USERNAME` | `CODEX_EMAIL__SMTP_USERNAME` | +| `CODEX_EMAIL_VERIFICATION_TOKEN_EXPIRY_HOURS` | `CODEX_EMAIL__VERIFICATION_TOKEN_EXPIRY_HOURS` | +| `CODEX_EMAIL_VERIFICATION_URL_BASE` | `CODEX_EMAIL__VERIFICATION_URL_BASE` | + +### `files` + +| Before | After | +| ------ | ----- | +| `CODEX_FILES_PLUGINS_DIR` | `CODEX_FILES__PLUGINS_DIR` | +| `CODEX_FILES_THUMBNAIL_DIR` | `CODEX_FILES__THUMBNAIL_DIR` | +| `CODEX_FILES_UPLOADS_DIR` | `CODEX_FILES__UPLOADS_DIR` | + +### `images` + +| Before | After | +| ------ | ----- | +| `CODEX_IMAGES_DECODE_CONCURRENCY` | `CODEX_IMAGES__DECODE_CONCURRENCY` | + +### `komga_api` + +| Before | After | +| ------ | ----- | +| `CODEX_KOMGA_API_ENABLED` | `CODEX_KOMGA_API__ENABLED` | +| `CODEX_KOMGA_API_PREFIX` | `CODEX_KOMGA_API__PREFIX` | + +### `koreader_api` + +| Before | After | +| ------ | ----- | +| `CODEX_KOREADER_API_ENABLED` | `CODEX_KOREADER_API__ENABLED` | + +### `logging` + +| Before | After | +| ------ | ----- | +| `CODEX_LOGGING_CONSOLE` | `CODEX_LOGGING__CONSOLE` | +| `CODEX_LOGGING_FILE` | `CODEX_LOGGING__FILE` | +| `CODEX_LOGGING_LEVEL` | `CODEX_LOGGING__LEVEL` | + +### `observability` + +| Before | After | +| ------ | ----- | +| `CODEX_OBSERVABILITY_BROWSER_ENABLED` | `CODEX_OBSERVABILITY__BROWSER__ENABLED` | +| `CODEX_OBSERVABILITY_BROWSER_PROXY_PATH` | `CODEX_OBSERVABILITY__BROWSER__PROXY_PATH` | +| `CODEX_OBSERVABILITY_BROWSER_SAMPLE_RATIO` | `CODEX_OBSERVABILITY__BROWSER__SAMPLE_RATIO` | +| `CODEX_OBSERVABILITY_ENABLED` | `CODEX_OBSERVABILITY__ENABLED` | +| `CODEX_OBSERVABILITY_METRICS_ENABLED` | `CODEX_OBSERVABILITY__METRICS__ENABLED` | +| `CODEX_OBSERVABILITY_METRICS_EXPORT_INTERVAL_MS` | `CODEX_OBSERVABILITY__METRICS__EXPORT_INTERVAL_MS` | +| `CODEX_OBSERVABILITY_OTLP_ENDPOINT` | `CODEX_OBSERVABILITY__OTLP__ENDPOINT` | +| `CODEX_OBSERVABILITY_OTLP_HEADERS` | `CODEX_OBSERVABILITY__OTLP__HEADERS` | +| `CODEX_OBSERVABILITY_OTLP_PROTOCOL` | `CODEX_OBSERVABILITY__OTLP__PROTOCOL` | +| `CODEX_OBSERVABILITY_OTLP_PROXY_ENDPOINT` | `CODEX_OBSERVABILITY__OTLP__PROXY_ENDPOINT` | +| `CODEX_OBSERVABILITY_OTLP_TIMEOUT_MS` | `CODEX_OBSERVABILITY__OTLP__TIMEOUT_MS` | +| `CODEX_OBSERVABILITY_SERVICE_NAME` | `CODEX_OBSERVABILITY__SERVICE_NAME` | +| `CODEX_OBSERVABILITY_TRACES_ENABLED` | `CODEX_OBSERVABILITY__TRACES__ENABLED` | +| `CODEX_OBSERVABILITY_TRACES_SAMPLE_RATIO` | `CODEX_OBSERVABILITY__TRACES__SAMPLE_RATIO` | + +### `pdf` + +| Before | After | +| ------ | ----- | +| `CODEX_PDF_CACHE_DIR` | `CODEX_PDF__CACHE_DIR` | +| `CODEX_PDF_CACHE_RENDERED_PAGES` | `CODEX_PDF__CACHE_RENDERED_PAGES` | +| `CODEX_PDF_JPEG_QUALITY` | `CODEX_PDF__JPEG_QUALITY` | +| `CODEX_PDF_PDFIUM_LIBRARY_PATH` | `CODEX_PDF__PDFIUM_LIBRARY_PATH` | +| `CODEX_PDF_RENDER_DPI` | `CODEX_PDF__RENDER_DPI` | + +### `pdf_handle_cache` + +| Before | After | +| ------ | ----- | +| `CODEX_PDF_HANDLE_CACHE_CAPACITY` | `CODEX_PDF_HANDLE_CACHE__CAPACITY` | +| `CODEX_PDF_HANDLE_CACHE_ENABLED` | `CODEX_PDF_HANDLE_CACHE__ENABLED` | +| `CODEX_PDF_HANDLE_CACHE_IDLE_TTL_MINUTES` | `CODEX_PDF_HANDLE_CACHE__IDLE_TTL_MINUTES` | +| `CODEX_PDF_HANDLE_CACHE_SWEEP_INTERVAL_SECONDS` | `CODEX_PDF_HANDLE_CACHE__SWEEP_INTERVAL_SECONDS` | + +### `plugins` + +| Before | After | +| ------ | ----- | +| `CODEX_PLUGINS_ALLOWED_COMMANDS` | `CODEX_PLUGINS__ALLOWED_COMMANDS` | + +### `rate_limit` + +| Before | After | +| ------ | ----- | +| `CODEX_RATE_LIMIT_ANONYMOUS_BURST` | `CODEX_RATE_LIMIT__ANONYMOUS_BURST` | +| `CODEX_RATE_LIMIT_ANONYMOUS_RPS` | `CODEX_RATE_LIMIT__ANONYMOUS_RPS` | +| `CODEX_RATE_LIMIT_AUTHENTICATED_BURST` | `CODEX_RATE_LIMIT__AUTHENTICATED_BURST` | +| `CODEX_RATE_LIMIT_AUTHENTICATED_RPS` | `CODEX_RATE_LIMIT__AUTHENTICATED_RPS` | +| `CODEX_RATE_LIMIT_BUCKET_TTL_SECS` | `CODEX_RATE_LIMIT__BUCKET_TTL_SECS` | +| `CODEX_RATE_LIMIT_CLEANUP_INTERVAL_SECS` | `CODEX_RATE_LIMIT__CLEANUP_INTERVAL_SECS` | +| `CODEX_RATE_LIMIT_ENABLED` | `CODEX_RATE_LIMIT__ENABLED` | +| `CODEX_RATE_LIMIT_EXEMPT_PATHS` | `CODEX_RATE_LIMIT__EXEMPT_PATHS` | + +### `scanner` + +| Before | After | +| ------ | ----- | +| `CODEX_SCANNER_MAX_CONCURRENT_SCANS` | `CODEX_SCANNER__MAX_CONCURRENT_SCANS` | + +### `scheduler` + +| Before | After | +| ------ | ----- | +| `CODEX_SCHEDULER_TIMEZONE` | `CODEX_SCHEDULER__TIMEZONE` | + +### `task` + +| Before | After | +| ------ | ----- | +| `CODEX_TASK_RUN_IN_PROCESS` | `CODEX_TASK__RUN_IN_PROCESS` | +| `CODEX_TASK_WORKER_COUNT` | `CODEX_TASK__WORKER_COUNT` | + +## Verifying + +After upgrading: + +```bash +codex config check +``` + +A clean run prints the resolved configuration with secrets redacted. As a +Kubernetes initContainer, `codex config check --strict --quiet` fails the pod +before the app container starts. diff --git a/docs/docs/observability.md b/docs/docs/observability.md index e3ee7984d..56472a2b1 100644 --- a/docs/docs/observability.md +++ b/docs/docs/observability.md @@ -25,11 +25,11 @@ Jaeger exposes its UI at [http://localhost:16686](http://localhost:16686). Hit a The env overrides live in `docker-compose.yml` under the `codex-dev` and `codex-dev-worker` services: ```yaml -CODEX_OBSERVABILITY_ENABLED: "true" -CODEX_OBSERVABILITY_SERVICE_NAME: codex -CODEX_OBSERVABILITY_OTLP_ENDPOINT: http://jaeger:4317 -CODEX_OBSERVABILITY_OTLP_PROTOCOL: grpc -CODEX_OBSERVABILITY_BROWSER_ENABLED: "true" # codex-dev only; enables RUM proxy +CODEX_OBSERVABILITY__ENABLED: "true" +CODEX_OBSERVABILITY__SERVICE_NAME: codex +CODEX_OBSERVABILITY__OTLP__ENDPOINT: http://jaeger:4317 +CODEX_OBSERVABILITY__OTLP__PROTOCOL: grpc +CODEX_OBSERVABILITY__BROWSER__ENABLED: "true" # codex-dev only; enables RUM proxy ``` `config/config.docker.yaml` itself ships with the `observability:` block commented out so a production deployment using the same config doesn't quietly start exporting telemetry — the dev override is intentionally local to the compose file. diff --git a/docs/docs/plugins/index.md b/docs/docs/plugins/index.md index e96363d25..1e51a7439 100644 --- a/docs/docs/plugins/index.md +++ b/docs/docs/plugins/index.md @@ -147,7 +147,7 @@ Plugin data is isolated per user. Each user-plugin connection has a unique `user Plugins run as **child processes** spawned by Codex, communicating over stdin/stdout via JSON-RPC. This provides process-level isolation: -- **Command allowlist**: Only approved commands can be used to launch plugins (`node`, `npx`, `python`, `python3`, `uv`, `uvx`, and paths under `/opt/codex/plugins/`). Custom commands can be allowed via the `CODEX_PLUGIN_ALLOWED_COMMANDS` environment variable. +- **Command allowlist**: Only approved commands can be used to launch plugins (`node`, `npx`, `python`, `python3`, `uv`, `uvx`, and paths under `/opt/codex/plugins/`). Custom commands can be allowed via the `CODEX_PLUGINS__ALLOWED_COMMANDS` environment variable. - **Environment variable blocklist**: Dangerous environment variables are stripped before spawning plugins, including `LD_*`, `DYLD_*`, `PATH`, `HOME`, `PYTHONPATH`, `NODE_PATH`, and others that could enable library injection or path manipulation. - **Request timeout**: Every JSON-RPC request has a **30-second timeout**. If a plugin hangs or becomes unresponsive, the request fails gracefully rather than blocking the server. - **Health monitoring**: Failed requests are tracked, and plugins that fail repeatedly are automatically disabled. diff --git a/docs/docs/third-party-apps.md b/docs/docs/third-party-apps.md index b8ac62451..1b62dfbfa 100644 --- a/docs/docs/third-party-apps.md +++ b/docs/docs/third-party-apps.md @@ -103,7 +103,7 @@ koreader_api: ### Via Environment Variables ```bash -CODEX_KOREADER_API_ENABLED=true +CODEX_KOREADER_API__ENABLED=true ``` After enabling, restart Codex and run a **deep scan** on your libraries to compute KOReader-compatible file hashes. @@ -124,8 +124,8 @@ komga_api: ### Via Environment Variables ```bash -CODEX_KOMGA_API_ENABLED=true -CODEX_KOMGA_API_PREFIX=komga +CODEX_KOMGA_API__ENABLED=true +CODEX_KOMGA_API__PREFIX=komga ``` ## API Endpoints diff --git a/docs/docs/troubleshooting.md b/docs/docs/troubleshooting.md index 2b401d1f3..dbbceb7ce 100644 --- a/docs/docs/troubleshooting.md +++ b/docs/docs/troubleshooting.md @@ -107,7 +107,7 @@ docker compose exec codex env | grep CODEX 4. **Enable debug logging temporarily**: ```bash - CODEX_LOGGING_LEVEL=debug docker compose up + CODEX_LOGGING__LEVEL=debug docker compose up ``` ### Server Hanging on Restart @@ -750,7 +750,7 @@ For detailed debugging: 3. **Enable debug logging**: ```bash - CODEX_LOGGING_LEVEL=debug docker compose up + CODEX_LOGGING__LEVEL=debug docker compose up ``` ### Common Error Codes diff --git a/docs/docs/users/oidc.md b/docs/docs/users/oidc.md index f298c0543..91334b357 100644 --- a/docs/docs/users/oidc.md +++ b/docs/docs/users/oidc.md @@ -81,30 +81,30 @@ All OIDC settings can be configured via environment variables: ```bash # Global settings -CODEX_AUTH_OIDC_ENABLED=true -CODEX_AUTH_OIDC_AUTO_CREATE_USERS=true -CODEX_AUTH_OIDC_DEFAULT_ROLE=reader -CODEX_AUTH_OIDC_REDIRECT_URI_BASE="https://codex.example.com" +CODEX_AUTH__OIDC__ENABLED=true +CODEX_AUTH__OIDC__AUTO_CREATE_USERS=true +CODEX_AUTH__OIDC__DEFAULT_ROLE=reader +CODEX_AUTH__OIDC__REDIRECT_URI_BASE="https://codex.example.com" # Provider-specific settings -CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_DISPLAY_NAME="Authentik" -CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_ISSUER_URL="https://authentik.example.com/application/o/codex/" -CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_CLIENT_ID="codex" -CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_CLIENT_SECRET="your-secret" -CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_CLIENT_SECRET_ENV="MY_OIDC_SECRET" -CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_SCOPES="email, profile, groups" -CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_GROUPS_CLAIM="groups" -CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_USERNAME_CLAIM="preferred_username" -CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_EMAIL_CLAIM="email" -CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_ACCEPTED_AUDIENCES="codex-client, other-trusted-client" +CODEX_AUTH__OIDC__PROVIDERS__AUTHENTIK__DISPLAY_NAME="Authentik" +CODEX_AUTH__OIDC__PROVIDERS__AUTHENTIK__ISSUER_URL="https://authentik.example.com/application/o/codex/" +CODEX_AUTH__OIDC__PROVIDERS__AUTHENTIK__CLIENT_ID="codex" +CODEX_AUTH__OIDC__PROVIDERS__AUTHENTIK__CLIENT_SECRET="your-secret" +CODEX_AUTH__OIDC__PROVIDERS__AUTHENTIK__CLIENT_SECRET_ENV="MY_OIDC_SECRET" +CODEX_AUTH__OIDC__PROVIDERS__AUTHENTIK__SCOPES="email, profile, groups" +CODEX_AUTH__OIDC__PROVIDERS__AUTHENTIK__GROUPS_CLAIM="groups" +CODEX_AUTH__OIDC__PROVIDERS__AUTHENTIK__USERNAME_CLAIM="preferred_username" +CODEX_AUTH__OIDC__PROVIDERS__AUTHENTIK__EMAIL_CLAIM="email" +CODEX_AUTH__OIDC__PROVIDERS__AUTHENTIK__ACCEPTED_AUDIENCES="codex-client, other-trusted-client" # Role mapping (comma-separated group names per role) -CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_ROLE_MAPPING_ADMIN="codex-admins, administrators" -CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_ROLE_MAPPING_MAINTAINER="codex-editors" -CODEX_AUTH_OIDC_PROVIDERS_AUTHENTIK_ROLE_MAPPING_READER="codex-users, users" +CODEX_AUTH__OIDC__PROVIDERS__AUTHENTIK__ROLE_MAPPING__ADMIN="codex-admins, administrators" +CODEX_AUTH__OIDC__PROVIDERS__AUTHENTIK__ROLE_MAPPING__MAINTAINER="codex-editors" +CODEX_AUTH__OIDC__PROVIDERS__AUTHENTIK__ROLE_MAPPING__READER="codex-users, users" ``` -Providers can also be created entirely via environment variables (no YAML needed). Setting `CODEX_AUTH_OIDC_PROVIDERS__ISSUER_URL` is sufficient to create a new provider entry. +Providers can also be created entirely via environment variables (no YAML needed). Setting `CODEX_AUTH__OIDC__PROVIDERS___ISSUER_URL` is sufficient to create a new provider entry. :::tip Use `client_secret_env` instead of `client_secret` to avoid storing secrets in configuration files: diff --git a/docs/sidebars.ts b/docs/sidebars.ts index f5b52517c..ce9adbcbf 100644 --- a/docs/sidebars.ts +++ b/docs/sidebars.ts @@ -11,6 +11,11 @@ const sidebars: SidebarsConfig = { collapsed: false, items: ["getting-started", "configuration"], }, + { + type: "category", + label: "Upgrading", + items: ["migration/v2-config"], + }, { type: "category", label: "Libraries & Scanning", From 173b3f2d3b05c32c5df1bcec4844125ff3b59743 Mon Sep 17 00:00:00 2001 From: Sylvain Cau Date: Sat, 15 Aug 2026 19:20:52 -0700 Subject: [PATCH 09/11] refactor(config)!: require typed environment values and drop the lenient parsers figment already parses typed values out of an environment string: `true`, `[a, b]`, `{k=v, k=v}`, numbers. The custom deserializers existed only to also accept the pre-2.0 spellings the old override layer hand-parsed, so an entire module was carrying compatibility with a scheme this release already breaks. It is deleted. What operators write changes with it: boolean 1 / 0 / yes / no / on / off -> true / false list a,b,c -> [a, b, c] map k1=v1,k2=v2 -> {k1=v1, k2=v2} Quote any entry containing a space or a comma, since those delimit entries. Numbers and plain strings are unaffected. One convenience survives, because losing it would be a regression rather than a simplification: an empty value still means "unset", since blanking a variable is how a setting gets switched off in compose files and Kubernetes manifests. That no longer needs a deserializer on every optional field, only a single normalization step after resolution. `codex config check` now reports a bad value instead of dying on it. Parsing failure used to abort the whole command, so one mistyped value hid every misspelled variable beside it, and those are exactly the pair worth seeing together. The resolved-config dump is suppressed when resolution failed, since printing the defaults would misrepresent what the server would actually run with. Serde stops at the first type error, so one is reported at a time, and the output says so rather than leaving that to be inferred from a second failure. Also repairs the Docker build, which this branch had broken. The starter template was pulled in with `include_str!` from `config/`, a directory `.dockerignore` excludes deliberately so operator configs are never baked into an image, so the crate failed to compile inside the image while every local build succeeded. The template is program data emitted by `codex config init` rather than an example to copy, so it moves inside the crate and the include becomes crate-relative. Verified by building with `config/` absent, which is what the image build sees. The migration guide gains a before-and-after table for value syntax, and the OIDC, rate-limit and OTLP examples are rewritten in bracket and brace form. --- .../codex-config/src}/codex.example.yaml | 0 crates/codex-config/src/de.rs | 504 ------------------ crates/codex-config/src/lib.rs | 1 - crates/codex-config/src/loader.rs | 41 +- crates/codex-config/src/types.rs | 77 ++- docs/docs/configuration.md | 23 +- docs/docs/migration/v2-config.md | 44 +- docs/docs/users/oidc.md | 10 +- src/commands/config.rs | 86 ++- 9 files changed, 187 insertions(+), 599 deletions(-) rename {config => crates/codex-config/src}/codex.example.yaml (100%) delete mode 100644 crates/codex-config/src/de.rs diff --git a/config/codex.example.yaml b/crates/codex-config/src/codex.example.yaml similarity index 100% rename from config/codex.example.yaml rename to crates/codex-config/src/codex.example.yaml diff --git a/crates/codex-config/src/de.rs b/crates/codex-config/src/de.rs deleted file mode 100644 index ce69d7c61..000000000 --- a/crates/codex-config/src/de.rs +++ /dev/null @@ -1,504 +0,0 @@ -//! Deserializers that accept the input shapes operators already use. -//! -//! Environment variables are strings, but the config tree holds bools, lists -//! and maps. The v1 override layer hand-parsed each of those at the point of -//! use; plain serde would now reject the same input. These deserializers keep -//! the accepted spellings working through the figment env provider. -//! -//! One deliberate behaviour change: v1 swallowed parse failures -//! (`if let Ok(v) = s.parse()`), so `CODEX_KOMGA_API_ENABLED=ture` silently -//! meant `false`. These reject unrecognized input instead. A typo in a -//! deployment's environment should stop the process, not quietly flip a -//! setting. - -use serde::de::{self, MapAccess, SeqAccess, Visitor}; -use serde::{Deserialize, Deserializer}; -use std::collections::HashMap; -use std::fmt; - -/// A bool from a bool, a number, or a string. -/// -/// figment parses `true` into a bool and `1` into a number before serde sees -/// them, so a field that must accept both has to handle both. -pub fn truthy_bool<'de, D>(deserializer: D) -> Result -where - D: Deserializer<'de>, -{ - deserializer.deserialize_any(TruthyBool) -} - -struct TruthyBool; - -impl<'de> Visitor<'de> for TruthyBool { - type Value = bool; - - fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.write_str("a boolean: true/false, 1/0, yes/no or on/off") - } - - fn visit_bool(self, value: bool) -> Result { - Ok(value) - } - - fn visit_u64(self, value: u64) -> Result { - match value { - 0 => Ok(false), - 1 => Ok(true), - other => Err(E::custom(format!("expected 0 or 1, found {other}"))), - } - } - - fn visit_i64(self, value: i64) -> Result { - match value { - 0 => Ok(false), - 1 => Ok(true), - other => Err(E::custom(format!("expected 0 or 1, found {other}"))), - } - } - - fn visit_str(self, value: &str) -> Result { - match value.trim().to_ascii_lowercase().as_str() { - "true" | "1" | "yes" | "on" => Ok(true), - "false" | "0" | "no" | "off" => Ok(false), - other => Err(E::custom(format!( - "expected true/false, 1/0, yes/no or on/off, found `{other}`" - ))), - } - } -} - -/// An optional bool with the same lenient spellings. -/// -/// Needed because `Option` does not route through [`truthy_bool`]: -/// serde hands the inner value to the option's own deserializer, so -/// `CODEX_AUTH__COOKIE_SECURE=1` would otherwise fail with "expected a -/// boolean". -pub fn optional_truthy_bool<'de, D>(deserializer: D) -> Result, D::Error> -where - D: Deserializer<'de>, -{ - struct OptionalTruthyBool; - - impl<'de> Visitor<'de> for OptionalTruthyBool { - type Value = Option; - - fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.write_str("a boolean, or nothing") - } - - fn visit_none(self) -> Result { - Ok(None) - } - - fn visit_unit(self) -> Result { - Ok(None) - } - - fn visit_bool(self, value: bool) -> Result { - Ok(Some(value)) - } - - fn visit_u64(self, value: u64) -> Result { - TruthyBool.visit_u64(value).map(Some) - } - - fn visit_str(self, value: &str) -> Result { - if value.is_empty() { - return Ok(None); - } - TruthyBool.visit_str(value).map(Some) - } - - fn visit_some>(self, inner: D2) -> Result { - inner.deserialize_any(TruthyBool).map(Some) - } - } - - deserializer.deserialize_option(OptionalTruthyBool) -} - -/// A list of strings from a real list or a comma-separated string. -/// -/// `CODEX_API__CORS_ORIGINS=https://a.example,https://b.example` has to mean -/// the same thing as a YAML sequence. -pub fn string_list<'de, D>(deserializer: D) -> Result, D::Error> -where - D: Deserializer<'de>, -{ - struct StringList; - - impl<'de> Visitor<'de> for StringList { - type Value = Vec; - - fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.write_str("a list of strings, or a comma-separated string") - } - - fn visit_str(self, value: &str) -> Result, E> { - Ok(split_csv(value)) - } - - // figment turns an all-digit value into a number before serde sees it, - // so a single numeric entry would otherwise fail to deserialize. - fn visit_u64(self, value: u64) -> Result, E> { - Ok(vec![value.to_string()]) - } - - fn visit_i64(self, value: i64) -> Result, E> { - Ok(vec![value.to_string()]) - } - - fn visit_seq>(self, mut seq: A) -> Result, A::Error> { - let mut out = Vec::new(); - while let Some(item) = seq.next_element::()? { - out.push(item); - } - Ok(out) - } - } - - deserializer.deserialize_any(StringList) -} - -/// A string map from a real map or a `k1=v1,k2=v2` string. -/// -/// Used for OTLP headers, where the whole set is conventionally supplied in -/// one variable. -pub fn string_map<'de, D>(deserializer: D) -> Result, D::Error> -where - D: Deserializer<'de>, -{ - deserializer.deserialize_any(StringMapVisitor) -} - -struct StringMapVisitor; - -impl<'de> Visitor<'de> for StringMapVisitor { - type Value = HashMap; - - fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.write_str("a map, or a comma-separated list of key=value pairs") - } - - fn visit_str(self, value: &str) -> Result { - let mut out = HashMap::new(); - for entry in split_csv(value) { - let (key, val) = entry - .split_once('=') - .ok_or_else(|| E::custom(format!("expected `key=value`, found `{entry}`")))?; - let key = key.trim(); - if key.is_empty() { - return Err(E::custom(format!("empty key in `{entry}`"))); - } - out.insert(key.to_string(), val.trim().to_string()); - } - Ok(out) - } - - fn visit_map>(self, mut map: A) -> Result { - let mut out = HashMap::new(); - while let Some((key, value)) = map.next_entry::()? { - out.insert(key, value); - } - Ok(out) - } -} - -/// An optional string map, empty meaning absent. -/// -/// `database.sqlite.pragmas` is `Option>`, so it needs the option -/// wrapper for the same reason `optional_truthy_bool` does: serde hands the -/// inner value to the option's deserializer, bypassing [`string_map`]. -pub fn optional_string_map<'de, D>( - deserializer: D, -) -> Result>, D::Error> -where - D: Deserializer<'de>, -{ - struct OptionalStringMap; - - impl<'de> Visitor<'de> for OptionalStringMap { - type Value = Option>; - - fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.write_str("a map, `key=value` pairs, or nothing") - } - - fn visit_none(self) -> Result { - Ok(None) - } - - fn visit_unit(self) -> Result { - Ok(None) - } - - fn visit_str(self, value: &str) -> Result { - if value.is_empty() { - return Ok(None); - } - StringMapVisitor.visit_str(value).map(Some) - } - - fn visit_map>(self, map: A) -> Result { - StringMapVisitor.visit_map(map).map(Some) - } - - fn visit_some>(self, inner: D2) -> Result { - inner.deserialize_any(StringMapVisitor).map(Some) - } - } - - deserializer.deserialize_option(OptionalStringMap) -} - -/// A map whose values are themselves comma-separated lists. -/// -/// OIDC role mapping is set one role at a time -/// (`CODEX_AUTH__OIDC__PROVIDERS__X__ROLE_MAPPING__ADMIN=grp-a,grp-b`), so the -/// values arrive as strings rather than sequences. -pub fn string_list_map<'de, D>(deserializer: D) -> Result>, D::Error> -where - D: Deserializer<'de>, -{ - struct StringListMap; - - impl<'de> Visitor<'de> for StringListMap { - type Value = HashMap>; - - fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.write_str("a map of names to string lists") - } - - fn visit_map>(self, mut map: A) -> Result { - #[derive(Deserialize)] - struct Entry(#[serde(deserialize_with = "string_list")] Vec); - - let mut out = HashMap::new(); - while let Some((key, Entry(value))) = map.next_entry::()? { - out.insert(key, value); - } - Ok(out) - } - } - - deserializer.deserialize_map(StringListMap) -} - -/// An optional string where empty means absent. -/// -/// v1's `env_string_opt` filtered empty values, so `CODEX_LOGGING__FILE=` -/// meant "no log file" rather than "log to a file named empty string". Keeping -/// that matters because unsetting a variable and blanking it are the same -/// gesture in most deployment tooling. -pub fn optional_string<'de, D>(deserializer: D) -> Result, D::Error> -where - D: Deserializer<'de>, -{ - struct OptionalString; - - impl<'de> Visitor<'de> for OptionalString { - type Value = Option; - - fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.write_str("a string, or nothing") - } - - fn visit_none(self) -> Result { - Ok(None) - } - - fn visit_unit(self) -> Result { - Ok(None) - } - - fn visit_str(self, value: &str) -> Result { - Ok(non_empty(value)) - } - - fn visit_some>(self, inner: D2) -> Result { - let value = String::deserialize(inner)?; - Ok(non_empty(&value)) - } - } - - deserializer.deserialize_option(OptionalString) -} - -fn non_empty(value: &str) -> Option { - (!value.is_empty()).then(|| value.to_string()) -} - -/// Split on commas, trimming each entry and dropping empties, so a trailing -/// comma or padded list does not produce blank members. -fn split_csv(value: &str) -> Vec { - value - .split(',') - .map(str::trim) - .filter(|s| !s.is_empty()) - .map(str::to_string) - .collect() -} - -#[cfg(test)] -mod tests { - use super::*; - use serde::Deserialize; - - #[derive(Debug, Deserialize)] - struct BoolHolder { - #[serde(deserialize_with = "truthy_bool")] - value: bool, - } - - #[derive(Debug, Deserialize)] - struct ListHolder { - #[serde(deserialize_with = "string_list")] - value: Vec, - } - - #[derive(Debug, Deserialize)] - struct MapHolder { - #[serde(deserialize_with = "string_map")] - value: HashMap, - } - - #[derive(Debug, Deserialize)] - struct OptHolder { - #[serde(deserialize_with = "optional_string")] - value: Option, - } - - fn parse_bool(yaml: &str) -> Result { - serde_yaml::from_str::(yaml).map(|h| h.value) - } - - #[test] - fn bools_accept_the_spellings_v1_accepted() { - for yaml in ["value: true", "value: 'true'", "value: 'TRUE'", "value: 1"] { - assert!(parse_bool(yaml).unwrap(), "{yaml} should be true"); - } - for yaml in [ - "value: false", - "value: 'false'", - "value: 'False'", - "value: 0", - ] { - assert!(!parse_bool(yaml).unwrap(), "{yaml} should be false"); - } - } - - #[test] - fn bools_accept_yes_no_and_on_off() { - assert!(parse_bool("value: 'yes'").unwrap()); - assert!(parse_bool("value: 'on'").unwrap()); - assert!(!parse_bool("value: 'no'").unwrap()); - assert!(!parse_bool("value: 'off'").unwrap()); - } - - /// v1 treated anything that was not `true`/`1` as `false`, so a typo - /// silently disabled a feature. This must fail instead. - #[test] - fn a_misspelled_bool_is_an_error_not_a_silent_false() { - let error = parse_bool("value: ture").unwrap_err().to_string(); - assert!( - error.contains("ture"), - "error should quote the input: {error}" - ); - - assert!( - parse_bool("value: 2").is_err(), - "only 0 and 1 are numeric bools" - ); - } - - #[test] - fn lists_accept_a_sequence_or_a_comma_separated_string() { - let from_seq = serde_yaml::from_str::("value:\n - a\n - b\n") - .unwrap() - .value; - let from_csv = serde_yaml::from_str::("value: 'a,b'") - .unwrap() - .value; - assert_eq!(from_seq, vec!["a", "b"]); - assert_eq!(from_csv, from_seq); - } - - #[test] - fn list_entries_are_trimmed_and_blanks_dropped() { - let parsed = serde_yaml::from_str::("value: ' a , b , '") - .unwrap() - .value; - assert_eq!(parsed, vec!["a", "b"]); - - let empty = serde_yaml::from_str::("value: ''") - .unwrap() - .value; - assert!(empty.is_empty()); - } - - /// figment parses an all-digit value as a number, so a one-element list - /// of digits must still work. - #[test] - fn lists_accept_a_bare_number() { - let parsed = serde_yaml::from_str::("value: 8080") - .unwrap() - .value; - assert_eq!(parsed, vec!["8080"]); - } - - #[test] - fn maps_accept_a_mapping_or_key_value_pairs() { - let from_map = serde_yaml::from_str::("value:\n a: '1'\n b: '2'\n") - .unwrap() - .value; - let from_pairs = serde_yaml::from_str::("value: 'a=1,b=2'") - .unwrap() - .value; - assert_eq!(from_map, from_pairs); - assert_eq!(from_pairs.get("a").map(String::as_str), Some("1")); - } - - #[test] - fn a_pair_without_an_equals_sign_is_an_error() { - let error = serde_yaml::from_str::("value: 'a=1,oops'") - .unwrap_err() - .to_string(); - assert!( - error.contains("oops"), - "error should quote the entry: {error}" - ); - } - - #[test] - fn a_value_may_itself_contain_an_equals_sign() { - let parsed = serde_yaml::from_str::("value: 'authorization=Bearer a=b'") - .unwrap() - .value; - assert_eq!( - parsed.get("authorization").map(String::as_str), - Some("Bearer a=b") - ); - } - - #[test] - fn an_empty_optional_string_is_absent() { - assert_eq!( - serde_yaml::from_str::("value: ''") - .unwrap() - .value, - None - ); - assert_eq!( - serde_yaml::from_str::("value: null") - .unwrap() - .value, - None - ); - assert_eq!( - serde_yaml::from_str::("value: /var/log/codex.log") - .unwrap() - .value, - Some("/var/log/codex.log".to_string()) - ); - } -} diff --git a/crates/codex-config/src/lib.rs b/crates/codex-config/src/lib.rs index 0d78eff7e..eedb2031b 100644 --- a/crates/codex-config/src/lib.rs +++ b/crates/codex-config/src/lib.rs @@ -3,7 +3,6 @@ //! Extracted from the monolithic `codex` crate as the first workspace leaf in //! the workspace-split plan. Has no dependencies on other Codex crates. -mod de; mod env_audit; mod keys; mod loader; diff --git a/crates/codex-config/src/loader.rs b/crates/codex-config/src/loader.rs index 484db72d6..367a10bbe 100644 --- a/crates/codex-config/src/loader.rs +++ b/crates/codex-config/src/loader.rs @@ -77,6 +77,7 @@ impl Config { .extract() .with_context(|| format!("failed to parse configuration from {}", path.display()))?; + config.blank_optionals_to_none(); config.root_paths_at_data_dir(&|key| overrides.find_value(key).is_ok()); config.validate()?; @@ -112,7 +113,12 @@ impl Config { /// moment of first boot. A container started once with /// `CODEX_DATABASE__POSTGRES__PASSWORD` wrote that password into the generated /// YAML in plaintext. -pub const STARTER_CONFIG_YAML: &str = include_str!("../../../config/codex.example.yaml"); +/// +/// Kept inside the crate rather than in `config/`, which `.dockerignore` +/// excludes so that operator configs are never baked into an image. A +/// compile-time `include_str!` reaching outside the crate breaks in any build +/// context that trims the repository, which is exactly what happened. +pub const STARTER_CONFIG_YAML: &str = include_str!("codex.example.yaml"); /// Write [`STARTER_CONFIG_YAML`] to `path`, creating parent directories. /// @@ -615,10 +621,10 @@ auth: // ---- input shapes the v1 override layer accepted ---- #[test] - fn bools_accept_one_and_zero_from_the_environment() { + fn bools_are_written_as_true_and_false() { Jail::expect_with(|jail| { - jail.set_env("CODEX_KOMGA_API__ENABLED", "1"); - jail.set_env("CODEX_API__CORS_ENABLED", "0"); + jail.set_env("CODEX_KOMGA_API__ENABLED", "true"); + jail.set_env("CODEX_API__CORS_ENABLED", "false"); let config = Config::load(jail.directory().join("absent.yaml")).unwrap(); @@ -629,9 +635,10 @@ auth: } /// v1 read `eq_ignore_ascii_case("true") || == "1"`, so a typo meant - /// `false` and the operator never found out. It is now an error. + /// `false` and the operator never found out. Anything that is not a + /// boolean is now an error, `1` included. #[test] - fn a_misspelled_bool_stops_startup() { + fn a_value_that_is_not_a_bool_stops_startup() { Jail::expect_with(|jail| { jail.set_env("CODEX_KOMGA_API__ENABLED", "ture"); let error = Config::load(jail.directory().join("absent.yaml")).unwrap_err(); @@ -644,13 +651,13 @@ auth: } #[test] - fn comma_separated_lists_come_through_the_environment() { + fn lists_use_bracket_syntax_in_the_environment() { Jail::expect_with(|jail| { jail.set_env( "CODEX_API__CORS_ORIGINS", - "https://a.example, https://b.example", + "[https://a.example, https://b.example]", ); - jail.set_env("CODEX_RATE_LIMIT__EXEMPT_PATHS", "/health,/metrics"); + jail.set_env("CODEX_RATE_LIMIT__EXEMPT_PATHS", "[/health, /metrics]"); let config = Config::load(jail.directory().join("absent.yaml")).unwrap(); @@ -680,11 +687,13 @@ auth: } #[test] - fn otlp_headers_accept_key_value_pairs() { + fn maps_use_brace_syntax_in_the_environment() { Jail::expect_with(|jail| { + // A value containing a space or comma must be quoted; those + // characters delimit entries. jail.set_env( "CODEX_OBSERVABILITY__OTLP__HEADERS", - "authorization=Bearer tok,x-tenant=acme", + r#"{authorization="Bearer tok", x-tenant=acme}"#, ); let config = Config::load(jail.directory().join("absent.yaml")).unwrap(); @@ -736,7 +745,7 @@ auth: ); jail.set_env( "CODEX_AUTH__OIDC__PROVIDERS__AUTHENTIK__ROLE_MAPPING__ADMIN", - "codex-admins, platform", + "[codex-admins, platform]", ); let config = Config::load(jail.directory().join("absent.yaml")).unwrap(); @@ -845,7 +854,7 @@ task: images: decode_concurrency: 9 plugins: - allowed_commands: deno,bun + allowed_commands: [deno, bun] database: run_migrations: false migration_wait_timeout_secs: 42 @@ -869,11 +878,11 @@ database: #[test] fn the_relocated_settings_are_settable_from_the_environment() { Jail::expect_with(|jail| { - jail.set_env("CODEX_AUTH__COOKIE_SECURE", "1"); + jail.set_env("CODEX_AUTH__COOKIE_SECURE", "true"); jail.set_env("CODEX_TASK__RUN_IN_PROCESS", "false"); jail.set_env("CODEX_IMAGES__DECODE_CONCURRENCY", "12"); - jail.set_env("CODEX_PLUGINS__ALLOWED_COMMANDS", "deno, bun"); - jail.set_env("CODEX_DATABASE__RUN_MIGRATIONS", "no"); + jail.set_env("CODEX_PLUGINS__ALLOWED_COMMANDS", "[deno, bun]"); + jail.set_env("CODEX_DATABASE__RUN_MIGRATIONS", "false"); jail.set_env("CODEX_DATABASE__MIGRATION_WAIT_TIMEOUT_SECS", "60"); let config = Config::load(jail.directory().join("absent.yaml")).unwrap(); diff --git a/crates/codex-config/src/types.rs b/crates/codex-config/src/types.rs index a1322e306..6cd00b5e9 100644 --- a/crates/codex-config/src/types.rs +++ b/crates/codex-config/src/types.rs @@ -10,7 +10,6 @@ pub struct TaskConfig { /// Set false for a web-only replica in a deployment that runs workers in /// their own pods. Note that SSE task progress only reaches the browser /// when workers share the process, unless a progress bridge is configured. - #[serde(deserialize_with = "crate::de::truthy_bool")] pub run_in_process: bool, /// Number of parallel task workers to process tasks from the queue @@ -25,7 +24,6 @@ pub struct TaskConfig { pub struct KomgaApiConfig { /// Enable Komga-compatible API endpoints /// When enabled, routes are mounted at /{prefix}/api/v1/* - #[serde(deserialize_with = "crate::de::truthy_bool")] pub enabled: bool, /// URL prefix for Komga API (default: "komga") @@ -45,7 +43,6 @@ fn default_komga_prefix() -> String { pub struct KoreaderApiConfig { /// Enable KOReader sync API endpoints /// When enabled, routes are mounted at /koreader/* - #[serde(deserialize_with = "crate::de::truthy_bool")] pub enabled: bool, } @@ -55,7 +52,6 @@ pub struct KoreaderApiConfig { #[serde(default)] pub struct RateLimitConfig { /// Enable rate limiting (default: true) - #[serde(deserialize_with = "crate::de::truthy_bool")] pub enabled: bool, /// Requests per second for anonymous users (default: 10) @@ -71,7 +67,6 @@ pub struct RateLimitConfig { pub authenticated_burst: u32, /// Glob patterns for paths exempt from rate limiting (e.g. `/api/v1/books/*/thumbnail`) - #[serde(deserialize_with = "crate::de::string_list")] pub exempt_paths: Vec, /// Cleanup interval in seconds for stale buckets (default: 60) @@ -131,23 +126,19 @@ pub struct OidcProviderConfig { /// OAuth2 client secret (optional if using client_secret_env) #[serde(default, skip_serializing_if = "Option::is_none")] - #[serde(deserialize_with = "crate::de::optional_string")] pub client_secret: Option, /// Environment variable name containing client secret #[serde(default, skip_serializing_if = "Option::is_none")] - #[serde(deserialize_with = "crate::de::optional_string")] pub client_secret_env: Option, /// Scopes to request (openid is always included) #[serde(default)] - #[serde(deserialize_with = "crate::de::string_list")] pub scopes: Vec, /// Group-to-role mapping: role -> [group names] /// e.g., {"admin": ["codex-admins"], "reader": ["codex-users"]} #[serde(default)] - #[serde(deserialize_with = "crate::de::string_list_map")] pub role_mapping: HashMap>, /// Claim containing groups (default: "groups") @@ -168,7 +159,6 @@ pub struct OidcProviderConfig { /// client). Resolution of the empty default happens at validator /// construction. #[serde(default)] - #[serde(deserialize_with = "crate::de::string_list")] pub accepted_audiences: Vec, } @@ -207,11 +197,9 @@ fn default_email_claim() -> String { #[serde(default)] pub struct OidcConfig { /// Enable OIDC authentication - #[serde(deserialize_with = "crate::de::truthy_bool")] pub enabled: bool, /// Auto-create users on first OIDC login - #[serde(deserialize_with = "crate::de::truthy_bool")] pub auto_create_users: bool, /// Default role for new OIDC users (if no group mapping matches) @@ -221,7 +209,6 @@ pub struct OidcConfig { /// e.g., "https://codex.example.com" or "http://localhost:8080" /// If not set, falls back to http://{application.host}:{application.port} #[serde(default, skip_serializing_if = "Option::is_none")] - #[serde(deserialize_with = "crate::de::optional_string")] pub redirect_uri_base: Option, /// Redirect targets a client may request at the end of an OIDC flow. @@ -235,7 +222,6 @@ pub struct OidcConfig { /// whatever target is used, so every entry here is a party trusted with the /// tokens of anyone who completes a sign-in. #[serde(default)] - #[serde(deserialize_with = "crate::de::string_list")] pub allowed_redirect_uris: Vec, /// Provider configurations keyed by provider name (e.g., "authentik", "keycloak") @@ -411,6 +397,39 @@ impl Config { Ok(()) } + /// Treat a blank optional string as absent. + /// + /// Values are typed now, so `CODEX_LOGGING__FILE=` would otherwise mean + /// "log to a file whose name is the empty string". Blanking a variable and + /// removing it are the same gesture in most deployment tooling, and the + /// pre-2.0 loader filtered empties, so this keeps that one convenience + /// without a custom deserializer on every optional field. + pub(crate) fn blank_optionals_to_none(&mut self) { + fn clear(slot: &mut Option) { + if slot.as_deref().is_some_and(str::is_empty) { + *slot = None; + } + } + + clear(&mut self.application.base_url); + clear(&mut self.logging.file); + clear(&mut self.email.verification_url_base); + clear(&mut self.pdf.pdfium_library_path); + clear(&mut self.observability.otlp.proxy_endpoint); + clear(&mut self.auth.oidc.redirect_uri_base); + + if let Some(postgres) = self.database.postgres.as_mut() { + clear(&mut postgres.ssl_root_cert); + clear(&mut postgres.ssl_client_cert); + clear(&mut postgres.ssl_client_key); + } + + for provider in self.auth.oidc.providers.values_mut() { + clear(&mut provider.client_secret); + clear(&mut provider.client_secret_env); + } + } + /// Root the per-kind data directories under `data_dir`. /// /// Every sub-path (`files.*_dir`, `pdf.cache_dir`, the SQLite file) defaults @@ -520,17 +539,11 @@ pub struct AuthConfig { /// Left unset it is off, which is what plain-HTTP development needs. Any /// deployment terminating TLS should set it to true, or the session cookie /// will also be sent over a plaintext downgrade. - #[serde( - default, - skip_serializing_if = "Option::is_none", - deserialize_with = "crate::de::optional_truthy_bool" - )] + #[serde(default, skip_serializing_if = "Option::is_none")] pub cookie_secure: Option, pub jwt_expiry_hours: u32, - #[serde(deserialize_with = "crate::de::truthy_bool")] pub refresh_token_enabled: bool, pub refresh_token_expiry_days: u32, - #[serde(deserialize_with = "crate::de::truthy_bool")] pub email_confirmation_required: bool, pub argon2_memory_cost: u32, pub argon2_time_cost: u32, @@ -568,12 +581,9 @@ impl Default for AuthConfig { #[serde(default)] pub struct ApiConfig { pub base_path: String, - #[serde(deserialize_with = "crate::de::truthy_bool")] pub enable_api_docs: bool, pub api_docs_path: String, - #[serde(deserialize_with = "crate::de::truthy_bool")] pub cors_enabled: bool, - #[serde(deserialize_with = "crate::de::string_list")] pub cors_origins: Vec, pub max_page_size: usize, } @@ -600,7 +610,6 @@ pub struct DatabaseConfig { /// /// Set false when a separate Job or init container owns migrations; this /// process then waits for the schema to be current instead of applying it. - #[serde(deserialize_with = "crate::de::truthy_bool")] pub run_migrations: bool, /// How long to wait for the database to accept connections, and for its @@ -772,15 +781,12 @@ pub struct PostgresConfig { /// CA certificate used to verify the server, required by `verify-ca` and /// `verify-full` unless the system trust store already has it. #[serde(default, skip_serializing_if = "Option::is_none")] - #[serde(deserialize_with = "crate::de::optional_string")] pub ssl_root_cert: Option, /// Client certificate and key, for mutual TLS. #[serde(default, skip_serializing_if = "Option::is_none")] - #[serde(deserialize_with = "crate::de::optional_string")] pub ssl_client_cert: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - #[serde(deserialize_with = "crate::de::optional_string")] pub ssl_client_key: Option, // Connection Pool Settings @@ -865,7 +871,6 @@ impl Default for PostgresConfig { #[serde(default)] pub struct SQLiteConfig { pub path: String, - #[serde(deserialize_with = "crate::de::optional_string_map")] pub pragmas: Option>, // Connection Pool Settings @@ -944,7 +949,6 @@ pub struct ApplicationConfig { /// Used as a fallback for OIDC redirect URIs and email verification links. /// If not set, falls back to http://{host}:{port} #[serde(default, skip_serializing_if = "Option::is_none")] - #[serde(deserialize_with = "crate::de::optional_string")] pub base_url: Option, } @@ -973,9 +977,7 @@ impl Default for ApplicationConfig { #[serde(default)] pub struct LoggingConfig { pub level: LogLevel, - #[serde(deserialize_with = "crate::de::truthy_bool")] pub console: bool, - #[serde(deserialize_with = "crate::de::optional_string")] pub file: Option, } @@ -1050,7 +1052,6 @@ impl Default for ImagesConfig { pub struct PluginsConfig { /// Executables plugins may spawn, in addition to the built-in runtimes /// (`node`, `npx`, `python`, `python3`, `uv`, `uvx`). - #[serde(deserialize_with = "crate::de::string_list")] pub allowed_commands: Vec, } @@ -1131,7 +1132,6 @@ pub struct EmailConfig { /// Base URL for email verification links (e.g., "https://codex.example.com") /// If not set, falls back to application.base_url, then http://{host}:{port} #[serde(default, skip_serializing_if = "Option::is_none")] - #[serde(deserialize_with = "crate::de::optional_string")] pub verification_url_base: Option, } @@ -1141,7 +1141,6 @@ pub struct EmailConfig { pub struct PdfConfig { /// Path to PDFium library (optional) /// If not set, will search current directory and system paths - #[serde(deserialize_with = "crate::de::optional_string")] pub pdfium_library_path: Option, /// Default render DPI (72-300, default 150) @@ -1152,7 +1151,6 @@ pub struct PdfConfig { pub jpeg_quality: u8, /// Enable rendered page caching (default: true) - #[serde(deserialize_with = "crate::de::truthy_bool")] pub cache_rendered_pages: bool, /// Directory for caching rendered PDF pages (default: data/cache) @@ -1179,7 +1177,6 @@ impl Default for PdfConfig { #[serde(default)] pub struct PdfHandleCacheConfig { /// Master switch. When false, every page render re-opens the PDF. - #[serde(deserialize_with = "crate::de::truthy_bool")] pub enabled: bool, /// Maximum number of open `PdfDocument` handles to keep resident. @@ -1273,7 +1270,6 @@ pub struct OtlpConfig { pub protocol: OtlpProtocol, /// Arbitrary headers attached to every export request (auth, tenancy). - #[serde(deserialize_with = "crate::de::string_map")] pub headers: HashMap, /// Per-export request timeout in milliseconds. @@ -1285,7 +1281,6 @@ pub struct OtlpConfig { /// the matching HTTP port (e.g. `:4318`) without changing the SDK /// transport. When unset or empty, the proxy falls back to `endpoint`. #[serde(default)] - #[serde(deserialize_with = "crate::de::optional_string")] pub proxy_endpoint: Option, } @@ -1320,7 +1315,6 @@ impl Default for OtlpConfig { pub struct ObservabilityTracesConfig { /// Enable trace export. Honored only when the parent `observability.enabled` /// is also true. - #[serde(deserialize_with = "crate::de::truthy_bool")] pub enabled: bool, /// Parent-based sampling ratio in `[0.0, 1.0]`. Values outside this range @@ -1343,7 +1337,6 @@ impl Default for ObservabilityTracesConfig { pub struct ObservabilityMetricsConfig { /// Enable metrics export. Honored only when the parent `observability.enabled` /// is also true. - #[serde(deserialize_with = "crate::de::truthy_bool")] pub enabled: bool, /// Periodic reader export interval in milliseconds. @@ -1369,7 +1362,6 @@ pub struct ObservabilityBrowserConfig { /// Independent of the backend `observability.enabled` flag because some /// operators want server-side observability without shipping spans from /// every browser tab. - #[serde(deserialize_with = "crate::de::truthy_bool")] pub enabled: bool, /// Path on the Codex server where the browser SDK POSTs OTLP batches. @@ -1399,7 +1391,6 @@ impl Default for ObservabilityBrowserConfig { #[serde(default)] pub struct ObservabilityConfig { /// Master switch. Must be `true` for any OTel work to happen. - #[serde(deserialize_with = "crate::de::truthy_bool")] pub enabled: bool, /// `service.name` resource attribute. Identifies this process in the diff --git a/docs/docs/configuration.md b/docs/docs/configuration.md index ff2c1024e..83cfb6193 100644 --- a/docs/docs/configuration.md +++ b/docs/docs/configuration.md @@ -835,19 +835,18 @@ All configuration options can be overridden with environment variables using the Configuration paths are converted to environment variables: - Use uppercase -- Replace dots with underscores +- Separate nesting levels with `__` (a single `_` still separates words inside one key) - Prefix with `CODEX_` -:::warning These names change in Codex 2.0 -Because a single `_` separates both nesting levels and the words inside a field -name, `CODEX_RATE_LIMIT__ANONYMOUS_RPS` is ambiguous: nothing in the name says -the section is `rate_limit` rather than `rate`. Codex 2.0 uses `__` between -nesting levels instead, so that variable becomes -`CODEX_RATE_LIMIT__ANONYMOUS_RPS`. +Values are typed. Booleans are `true`/`false`, lists are `[a, b]`, and maps are +`{key=value, key=value}`; quote any entry containing a space or a comma. An +empty value means "unset". Anything that does not parse stops the server rather +than being silently discarded. -Run `codex config check` to see the new name for every variable you currently -set. **Do not rename anything until you upgrade**: this version does not read -the new spelling, and a variable renamed early is silently ignored. +:::info Upgrading from 1.x +These names changed in Codex 2.0, and the old flat spelling is no longer read. +Codex refuses to start when it sees one, listing each with its replacement. See +the [upgrade guide](./migration/v2-config.md). ::: | Config Path | Environment Variable | @@ -922,7 +921,7 @@ CODEX_RATE_LIMIT__ANONYMOUS_RPS=10 CODEX_RATE_LIMIT__ANONYMOUS_BURST=50 CODEX_RATE_LIMIT__AUTHENTICATED_RPS=50 CODEX_RATE_LIMIT__AUTHENTICATED_BURST=200 -CODEX_RATE_LIMIT__EXEMPT_PATHS=/health,/api/v1/events +CODEX_RATE_LIMIT__EXEMPT_PATHS='[/health, /api/v1/events]' CODEX_RATE_LIMIT__CLEANUP_INTERVAL_SECS=60 CODEX_RATE_LIMIT__BUCKET_TTL_SECS=300 @@ -931,7 +930,7 @@ CODEX_OBSERVABILITY__ENABLED=true CODEX_OBSERVABILITY__SERVICE_NAME=codex CODEX_OBSERVABILITY__OTLP__ENDPOINT=http://localhost:4317 CODEX_OBSERVABILITY__OTLP__PROTOCOL=grpc -CODEX_OBSERVABILITY__OTLP__HEADERS=signoz-access-token=abc123,x-tenant=production +CODEX_OBSERVABILITY__OTLP__HEADERS='{signoz-access-token=abc123, x-tenant=production}' CODEX_OBSERVABILITY__OTLP__TIMEOUT_MS=5000 CODEX_OBSERVABILITY__TRACES__ENABLED=true CODEX_OBSERVABILITY__TRACES__SAMPLE_RATIO=0.1 diff --git a/docs/docs/migration/v2-config.md b/docs/docs/migration/v2-config.md index 7ae6a0836..7da65039a 100644 --- a/docs/docs/migration/v2-config.md +++ b/docs/docs/migration/v2-config.md @@ -72,9 +72,42 @@ instead of the environment. ## Other behaviour changes -**A bad value now stops the server.** The old override layer discarded values -it could not parse, so `CODEX_KOMGA_API_ENABLED=ture` quietly meant `false`. -Unrecognized input is now an error naming the variable. +**Values are typed, and a bad one stops the server.** The old override layer +hand-parsed each value and discarded anything it could not read, so +`CODEX_KOMGA_API_ENABLED=ture` quietly meant `false`. Values are now parsed by +type, and anything that does not fit is an error naming the variable. + +This is the second thing to change when you upgrade, after the names: + +| Type | Before | Now | +| ---- | ------ | --- | +| boolean | `1`, `0`, `yes`, `no`, `on`, `off`, `true`, `false` | `true` or `false` | +| list | `a,b,c` | `[a, b, c]` | +| map | `k1=v1,k2=v2` | `{k1=v1, k2=v2}` | +| number | unchanged | unchanged | +| string | unchanged | unchanged | + +```bash +# before +CODEX_API_CORS_ORIGINS=https://a.example,https://b.example +CODEX_OBSERVABILITY_OTLP_HEADERS=authorization=Bearer tok,x-tenant=acme +CODEX_KOMGA_API_ENABLED=1 + +# now +CODEX_API__CORS_ORIGINS='[https://a.example, https://b.example]' +CODEX_OBSERVABILITY__OTLP__HEADERS='{authorization="Bearer tok", x-tenant=acme}' +CODEX_KOMGA_API__ENABLED=true +``` + +Quote a map or list value that contains a space or a comma, since those +delimit entries. + +An empty value still means "unset", so blanking a variable to turn a setting +off keeps working. + +Parsing stops at the first bad value, so `codex config check` reports one type +error at a time. The variable-name checks still run alongside it, so you will +not have to fix them one restart apart. **Startup no longer writes a config file.** Codex used to serialize its defaults to disk when the file was missing, which produced an uncommented dump @@ -82,11 +115,6 @@ and, because those defaults were read from the environment, could capture a database password in plaintext. Run `codex config init` for a commented starter instead. -**Values keep the shapes you already use.** Booleans still accept -`true`/`false`, `1`/`0`, `yes`/`no` and `on`/`off`. Lists still accept a -comma-separated string. `CODEX_OBSERVABILITY__OTLP__HEADERS` still accepts -`k1=v1,k2=v2`. An empty value still means "unset". - ## New in this release **Local overlay.** A `codex.local.yaml` beside your config is merged on top of diff --git a/docs/docs/users/oidc.md b/docs/docs/users/oidc.md index 91334b357..f219e5f4e 100644 --- a/docs/docs/users/oidc.md +++ b/docs/docs/users/oidc.md @@ -92,16 +92,16 @@ CODEX_AUTH__OIDC__PROVIDERS__AUTHENTIK__ISSUER_URL="https://authentik.example.co CODEX_AUTH__OIDC__PROVIDERS__AUTHENTIK__CLIENT_ID="codex" CODEX_AUTH__OIDC__PROVIDERS__AUTHENTIK__CLIENT_SECRET="your-secret" CODEX_AUTH__OIDC__PROVIDERS__AUTHENTIK__CLIENT_SECRET_ENV="MY_OIDC_SECRET" -CODEX_AUTH__OIDC__PROVIDERS__AUTHENTIK__SCOPES="email, profile, groups" +CODEX_AUTH__OIDC__PROVIDERS__AUTHENTIK__SCOPES="[email, profile, groups]" CODEX_AUTH__OIDC__PROVIDERS__AUTHENTIK__GROUPS_CLAIM="groups" CODEX_AUTH__OIDC__PROVIDERS__AUTHENTIK__USERNAME_CLAIM="preferred_username" CODEX_AUTH__OIDC__PROVIDERS__AUTHENTIK__EMAIL_CLAIM="email" -CODEX_AUTH__OIDC__PROVIDERS__AUTHENTIK__ACCEPTED_AUDIENCES="codex-client, other-trusted-client" +CODEX_AUTH__OIDC__PROVIDERS__AUTHENTIK__ACCEPTED_AUDIENCES="[codex-client, other-trusted-client]" # Role mapping (comma-separated group names per role) -CODEX_AUTH__OIDC__PROVIDERS__AUTHENTIK__ROLE_MAPPING__ADMIN="codex-admins, administrators" -CODEX_AUTH__OIDC__PROVIDERS__AUTHENTIK__ROLE_MAPPING__MAINTAINER="codex-editors" -CODEX_AUTH__OIDC__PROVIDERS__AUTHENTIK__ROLE_MAPPING__READER="codex-users, users" +CODEX_AUTH__OIDC__PROVIDERS__AUTHENTIK__ROLE_MAPPING__ADMIN="[codex-admins, administrators]" +CODEX_AUTH__OIDC__PROVIDERS__AUTHENTIK__ROLE_MAPPING__MAINTAINER="[codex-editors]" +CODEX_AUTH__OIDC__PROVIDERS__AUTHENTIK__ROLE_MAPPING__READER="[codex-users, users]" ``` Providers can also be created entirely via environment variables (no YAML needed). Setting `CODEX_AUTH__OIDC__PROVIDERS___ISSUER_URL` is sufficient to create a new provider entry. diff --git a/src/commands/config.rs b/src/commands/config.rs index 02f739729..e5abee927 100644 --- a/src/commands/config.rs +++ b/src/commands/config.rs @@ -51,17 +51,35 @@ pub fn config_command(config_path: PathBuf, command: ConfigSubcommand) -> Result } fn check(config_path: &Path, strict: bool, quiet: bool) -> Result<()> { - // `resolve` rather than `load`: the point is to list every problem, - // and `load` refuses to start on the first one. - let config = resolve_config(config_path)?; + // `resolve` rather than `load`: the point is to list every problem, and + // `load` refuses to start on the first one. + // + // A value of the wrong type fails resolution outright, so there is no + // config to audit against. Report it and carry on with the name checks + // rather than returning, because a mistyped value and a misspelled + // variable are exactly the pair an operator wants to see together. + let (config, type_error) = match resolve_config(config_path) { + Ok(config) => (config, None), + Err(error) => (Config::default(), Some(format!("{error:#}"))), + }; + let findings = audit_env_with_config(&config); - let report = build_report(config_path, &config, &findings, quiet)?; + let report = build_report( + config_path, + &config, + &findings, + type_error.as_deref(), + // The resolved config is the defaults when resolution failed, so + // printing it would be a lie. + quiet || type_error.is_some(), + )?; print!("{report}"); - // Match what the server will do: a fatal finding stops it, so `check` - // fails too. `--strict` additionally fails on warnings. - let failed = findings.iter().any(Finding::is_fatal) || (strict && !findings.is_empty()); + // Match what the server will do: it refuses to start on either of these. + let failed = type_error.is_some() + || findings.iter().any(Finding::is_fatal) + || (strict && !findings.is_empty()); if failed { std::process::exit(1); } @@ -76,6 +94,7 @@ fn build_report( config_path: &Path, config: &Config, findings: &[Finding], + type_error: Option<&str>, quiet: bool, ) -> Result { use std::fmt::Write as _; @@ -106,7 +125,23 @@ fn build_report( .filter(|f| matches!(f, Finding::Unknown { .. })) .collect(); - if findings.is_empty() { + if let Some(error) = type_error { + writeln!(out, "\nERROR: a value could not be parsed:\n")?; + for line in error.lines() { + writeln!(out, " {line}")?; + } + writeln!(out, "\n Values are typed. Booleans are `true`/`false`,")?; + writeln!( + out, + " lists are `[a, b]`, maps are `{{key=value, key=value}}`." + )?; + writeln!( + out, + " Parsing stops at the first bad value, so fix this one and run again." + )?; + } + + if findings.is_empty() && type_error.is_none() { writeln!(out, "\n No environment variable problems found.")?; } @@ -197,6 +232,7 @@ mod tests { Path::new("config/codex.yaml"), &Config::default(), findings, + None, quiet, ) .unwrap() @@ -268,6 +304,36 @@ mod tests { assert!(text.contains("not a Codex setting; ignored")); } + /// A mistyped value must not hide the misspelled variable next to it. + #[test] + fn a_type_error_is_reported_alongside_name_findings() { + let text = build_report( + Path::new("config/codex.yaml"), + &Config::default(), + &[legacy( + "CODEX_TASK_WORKER_COUNT", + "CODEX_TASK__WORKER_COUNT", + "task.worker_count", + )], + Some( + "invalid type: found string \"x\", expected a boolean for key \"API.CORS_ENABLED\"", + ), + true, + ) + .unwrap(); + + assert!(text.contains("a value could not be parsed")); + assert!(text.contains("API.CORS_ENABLED")); + assert!( + text.contains("`[a, b]`"), + "should show the list syntax: {text}" + ); + assert!( + text.contains("CODEX_TASK__WORKER_COUNT"), + "the name finding must still appear: {text}" + ); + } + #[test] fn a_clean_environment_reports_nothing_to_fix() { assert!(report(&[], true).contains("No environment variable problems found.")); @@ -283,7 +349,7 @@ mod tests { fn the_printed_config_hides_secrets() { let mut config = Config::default(); config.auth.jwt_secret = "a-real-signing-secret".to_string(); - let text = build_report(Path::new("config/codex.yaml"), &config, &[], false).unwrap(); + let text = build_report(Path::new("config/codex.yaml"), &config, &[], None, false).unwrap(); assert!(!text.contains("a-real-signing-secret")); assert!(text.contains(codex_config::REDACTED)); @@ -294,7 +360,7 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let missing = dir.path().join("nope.yaml"); - let text = build_report(&missing, &Config::default(), &[], true).unwrap(); + let text = build_report(&missing, &Config::default(), &[], None, true).unwrap(); assert!(text.contains("not found, using defaults")); assert!(!missing.exists(), "check must not create the config file"); From 1092e9ae86633cb0ba95ef1b41476acd33d22088 Mon Sep 17 00:00:00 2001 From: Sylvain Cau Date: Sat, 15 Aug 2026 19:43:27 -0700 Subject: [PATCH 10/11] fix(config): ignore local overlays and keep the starter template in config/ Two problems with files under config/, both introduced by this branch. `.gitignore` had no rule for `.local.`. That is the file the new overlay support tells operators to put secrets in, so the one file guaranteed to hold a credential was also the one guaranteed to be committed. Local overlays are now ignored. The starter template moves back to config/, where it sits beside the other examples and can simply be copied, which is the point of shipping an example at all. It was moved into the crate to unbreak the Docker build, but that fixed the symptom and gave up the property worth having. The build broke because `.dockerignore` excludes config/ so that operator configs are never baked into an image, and the template is pulled in with `include_str!` at compile time. That exclusion has been in place for months and was harmless until this branch added the repository's first compile-time dependency on a file, which happened to point into the excluded directory. Nothing local could catch it: every local build runs in a full checkout, and only the Docker build sees a trimmed context. The narrow fix is an explicit exception for that one file, so the template reaches the build context and every other config file stays out. Re-including a path inside an excluded directory is the sort of thing that silently does nothing, so it was verified with a throwaway image that copies the context and lists it: config/ arrives containing the template and nothing else. --- .dockerignore | 3 +++ .gitignore | 6 ++++++ {crates/codex-config/src => config}/codex.example.yaml | 0 crates/codex-config/src/loader.rs | 10 +++++----- 4 files changed, 14 insertions(+), 5 deletions(-) rename {crates/codex-config/src => config}/codex.example.yaml (100%) diff --git a/.dockerignore b/.dockerignore index 634664005..8f567095f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -79,6 +79,9 @@ codex.yaml # Config files (mounted as volumes, not needed in image) config/ +# ...except the starter template, which is compiled into the binary by +# `codex-config` and so must exist in the build context. +!config/codex.example.yaml # Docusaurus files docs/ diff --git a/.gitignore b/.gitignore index 2e6be8ae2..e9dee3799 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,9 @@ plugins/**/openapi.json seed-config.yaml .playwright-mcp qa-*.png + +# Local config overlays: `.local.` sits beside a committed config +# and is where secrets and per-host values go. Never commit one. +*.local.yaml +*.local.yml +*.local.toml diff --git a/crates/codex-config/src/codex.example.yaml b/config/codex.example.yaml similarity index 100% rename from crates/codex-config/src/codex.example.yaml rename to config/codex.example.yaml diff --git a/crates/codex-config/src/loader.rs b/crates/codex-config/src/loader.rs index 367a10bbe..f134a4b6b 100644 --- a/crates/codex-config/src/loader.rs +++ b/crates/codex-config/src/loader.rs @@ -114,11 +114,11 @@ impl Config { /// `CODEX_DATABASE__POSTGRES__PASSWORD` wrote that password into the generated /// YAML in plaintext. /// -/// Kept inside the crate rather than in `config/`, which `.dockerignore` -/// excludes so that operator configs are never baked into an image. A -/// compile-time `include_str!` reaching outside the crate breaks in any build -/// context that trims the repository, which is exactly what happened. -pub const STARTER_CONFIG_YAML: &str = include_str!("codex.example.yaml"); +/// Lives in `config/` so it sits next to the other examples and can simply be +/// copied. `.dockerignore` excludes that directory so operator configs are +/// never baked into an image, with an explicit exception for this one file +/// because the build needs it at compile time. +pub const STARTER_CONFIG_YAML: &str = include_str!("../../../config/codex.example.yaml"); /// Write [`STARTER_CONFIG_YAML`] to `path`, creating parent directories. /// From 058ce01575883c35354050d0b3ad6e144c130151 Mon Sep 17 00:00:00 2001 From: Sylvain Cau Date: Sat, 15 Aug 2026 20:38:51 -0700 Subject: [PATCH 11/11] test(config): check every key in the starter template is a real setting The template was covered by two tests that together proved very little. One parsed it, but almost every key in a starter template is commented out, so it exercised a handful of lines. The other checked that section names appeared as substrings, which a mention in prose would satisfy. That left the case that actually bites unguarded: a typo or a stale name in a commented key. Those are precisely the lines an operator uncomments, and a key that does not exist is silently ignored by the loader, so the setting has no effect and nothing explains why. The new test un-comments as it walks, builds dotted paths from indentation, and requires each one to be a real setting. Reading a heavily commented file this way means telling YAML from prose, and the discriminator is that a section carries no value of its own while a sentence always does. That keeps `exception: a list in the overlay ...` out of the results while still catching a section that has been renamed or removed. It checks one direction deliberately. A setting missing from the template is harmless, since the reference documents the whole surface and `config check` prints the resolved config, and requiring every key would turn a starter into a reference and put a documentation chore on every new field. Writing it exposed a bug in the key registry: map containers were filed as exact paths even when the path already contained a wildcard, so `auth.oidc.providers.*.role_mapping` sat in the set the environment classifier normalizes over to suggest replacements, where it could only have produced a suggestion with a literal `*` in the variable name. Paths now go through one routine that files each by its own shape. Also documents `ssl_client_cert` and `ssl_client_key`, which were added with the other PostgreSQL TLS settings but never reached the template. Commented out like the rest of the postgres block: an explicitly written value is not the same as an absent one here, and shipping `ssl_mode` uncommented would demand certificate verification from every copy of the file. --- config/codex.example.yaml | 3 + crates/codex-config/src/keys.rs | 36 ++++++----- crates/codex-config/src/loader.rs | 100 ++++++++++++++++++++++++++++++ 3 files changed, 124 insertions(+), 15 deletions(-) diff --git a/config/codex.example.yaml b/config/codex.example.yaml index c119e4a9a..cd022e76f 100644 --- a/config/codex.example.yaml +++ b/config/codex.example.yaml @@ -52,6 +52,9 @@ database: # # plaintext, both silently. Use verify-full for anything remote. # ssl_mode: verify-full # disable|allow|prefer|require|verify-ca|verify-full # ssl_root_cert: /etc/ssl/certs/postgres-ca.crt + # # Client certificate and key, only for mutual TLS. + # ssl_client_cert: /etc/ssl/certs/codex-client.crt + # ssl_client_key: /etc/ssl/private/codex-client.key # # Per process, not per deployment. Every replica, worker and job opens # # its own pool against the server's shared max_connections. # max_connections: 25 diff --git a/crates/codex-config/src/keys.rs b/crates/codex-config/src/keys.rs index d36a62677..d1cb99cff 100644 --- a/crates/codex-config/src/keys.rs +++ b/crates/codex-config/src/keys.rs @@ -53,6 +53,18 @@ impl KeyRegistry { self.exact.iter().chain(self.wildcard.iter()) } + /// File a path in whichever set matches its shape. + fn insert(&mut self, path: &str) { + if path.is_empty() { + return; + } + if path.split('.').any(|segment| segment == "*") { + self.wildcard.insert(path.to_string()); + } else { + self.exact.insert(path.to_string()); + } + } + /// Whether `path` names a real setting. Wildcard segments match any single /// non-empty path segment. pub fn contains(&self, path: &str) -> bool { @@ -94,12 +106,15 @@ fn build_registry() -> KeyRegistry { fn walk(value: &Value, path: &str, out: &mut KeyRegistry) { match value { Value::Object(fields) => { - // A map is settable as a whole, not only key by key: - // `CODEX_OBSERVABILITY_OTLP_HEADERS` takes the entire header set as - // `k1=v1,k2=v2`. Record the container alongside the per-key - // wildcard so that form is recognized too. + // A map is settable as a whole, not only key by key, so record the + // container alongside the per-key wildcard. It belongs in whichever + // set matches its own shape: a container that already sits under a + // wildcard (`auth.oidc.providers.*.role_mapping`) is not an exact + // path, and filing it as one puts a `*` into the set the + // environment classifier normalizes over, where it can only + // produce nonsense suggestions. if fields.contains_key(MAP_PROBE_KEY) && !path.is_empty() { - out.exact.insert(path.to_string()); + out.insert(path); } for (name, child) in fields { let segment = if name == MAP_PROBE_KEY { "*" } else { name }; @@ -111,16 +126,7 @@ fn walk(value: &Value, path: &str, out: &mut KeyRegistry) { walk(child, &child_path, out); } } - _ => { - if path.is_empty() { - return; - } - if path.split('.').any(|s| s == "*") { - out.wildcard.insert(path.to_string()); - } else { - out.exact.insert(path.to_string()); - } - } + _ => out.insert(path), } } diff --git a/crates/codex-config/src/loader.rs b/crates/codex-config/src/loader.rs index f134a4b6b..637064737 100644 --- a/crates/codex-config/src/loader.rs +++ b/crates/codex-config/src/loader.rs @@ -925,6 +925,106 @@ database: }); } + /// Every key the starter template names must be a real setting. + /// + /// The parse test above only exercises the handful of lines that are + /// uncommented, so a typo in any of the commented ones goes unnoticed, and + /// those are the lines operators uncomment. A key that does not exist is + /// silently ignored by the loader, so the failure is invisible: the + /// operator sets it, nothing happens, and nothing says why. + /// + /// The reverse direction is deliberately not checked. A setting missing + /// from the template is harmless, since the reference documentation covers + /// the whole surface and `config check` prints the resolved config, and + /// requiring every key here would turn a starter into a reference and put + /// a documentation chore on every new field. + #[test] + fn every_key_in_the_starter_template_is_a_real_setting() { + let registry = crate::registry(); + let sections: std::collections::BTreeSet<&str> = registry + .all() + .filter_map(|key| key.split('.').next()) + .collect(); + + // (indent, key) for each open level, innermost last. + let mut stack: Vec<(usize, String)> = Vec::new(); + let mut unknown: Vec = Vec::new(); + + for line in STARTER_CONFIG_YAML.lines() { + // A commented key still counts: uncommenting it is the intended + // use, so it has to be correct now. + let uncommented = match line.trim_start().strip_prefix('#') { + Some(rest) => { + let indent = line.len() - line.trim_start().len(); + format!( + "{}{}", + " ".repeat(indent), + rest.strip_prefix(' ').unwrap_or(rest) + ) + } + None => line.to_string(), + }; + + let indent = uncommented.len() - uncommented.trim_start().len(); + let Some((name, value)) = uncommented.trim_start().split_once(':') else { + continue; + }; + if name.is_empty() + || !name + .chars() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '_') + { + continue; + } + + while stack.last().is_some_and(|(open, _)| *open >= indent) { + stack.pop(); + } + + // Prose is full of sentences that look like `word: ...`, so the + // top level needs a way to tell one from a section. A section + // carries no value of its own; a sentence always does. That keeps + // `exception: a list in the overlay ...` out while still catching a + // section that has been renamed or removed. + if stack.is_empty() { + let is_prose = indent > 0 || (!value.trim().is_empty() && !sections.contains(name)); + if is_prose { + continue; + } + } + + stack.push((indent, name.to_string())); + let path = stack + .iter() + .map(|(_, key)| key.as_str()) + .collect::>() + .join("."); + + // A section is any strict prefix of a real key, comparing `*` + // segments loosely so a concrete map key such as + // `auth.oidc.providers.authentik` matches + // `auth.oidc.providers.*.issuer_url`. + let segments: Vec<&str> = path.split('.').collect(); + let is_section = registry.all().any(|key| { + let key_segments: Vec<&str> = key.split('.').collect(); + key_segments.len() > segments.len() + && key_segments + .iter() + .zip(segments.iter()) + .all(|(k, p)| *k == "*" || k == p) + }); + + if !registry.contains(&path) && !is_section { + unknown.push(path); + } + } + + assert!( + unknown.is_empty(), + "the starter template names settings that do not exist: {unknown:#?}" + ); + } + /// It documents the surface, so a new section that never reaches it is a /// section operators cannot discover. #[test]