Warm serverless: stateless writers, manifest refresh, lazy attach (roadmap Phases 0-3)

.env.example

@@ -59,6 +59,27 @@ RUST_LOG=compass=info
 # AZURE_STORAGE_CONNECTION_STRING=
 # AZURE_STORAGE_SAS_KEY=
 
+# ── Warm serverless (cloud mode only) ───────────────────────────────────────
+# Node role: `full` (default — serve reads + writes with local indexes) or
+# `writer` (durable-append-only: no local indexes, no read serving, instant
+# boot). Writers validate against the bucket's collection config.
+# COMPASS_ROLE=full
+
+# Seconds between manifest refreshes (convergence with other nodes' writes).
+# Default 5; 0 disables the background refresher.
+# COMPASS_REFRESH_INTERVAL=5
+
+# Lazy attach: register bucket collections at boot and attach (rebuild local
+# indexes) on first request instead of eagerly. Default false.
+# COMPASS_LAZY_ATTACH=false
+
+# Max simultaneously-attached collections when lazy attach is on (LRU detach
+# past the budget; detached collections re-attach on demand). 0 = unbounded.
+# COMPASS_MAX_ATTACHED=0
+
 # ── Telemetry (anonymous; opt out) ──────────────────────────────────────────
+# Global in-flight request cap (backpressure). Unset = effectively unlimited.
+# COMPASS_MAX_CONCURRENCY=1024
+
 # COMPASS_TELEMETRY=off
 # DO_NOT_TRACK=1

.github/workflows/ci.yml

@@ -57,6 +57,71 @@ jobs:
           sudo apt-get install -y cmake pkg-config libssl-dev
       - run: cargo test --workspace --exclude compass-vector-gpu
 
+  # Cloud-feature coverage: the object-storage build + the real-S3 integration
+  # tests against MinIO. Not in the required-checks list (new job), but a
+  # failure here still blocks review attention.
+  test-cloud:
+    runs-on: ubuntu-24.04
+    env:
+      COMPASS_TEST_S3_BUCKET: compass-data
+      COMPASS_S3_ENDPOINT: http://localhost:9000
+      COMPASS_S3_ALLOW_HTTP: "true"
+      AWS_ACCESS_KEY_ID: minioadmin
+      AWS_SECRET_ACCESS_KEY: minioadmin
+      AWS_DEFAULT_REGION: us-east-1
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+      - run: |
+          sudo apt-get update
+          sudo apt-get install -y cmake pkg-config libssl-dev
+      # MinIO as a plain container (service containers can't override the
+      # image command, and minio/minio needs `server /data`). Same images as
+      # docker-compose.minio.yml.
+      - run: |
+          docker run -d --name minio -p 9000:9000 \
+            -e MINIO_ROOT_USER=minioadmin -e MINIO_ROOT_PASSWORD=minioadmin \
+            minio/minio:latest server /data
+          for i in $(seq 1 30); do
+            curl -sf http://localhost:9000/minio/health/live && break
+            sleep 1
+          done
+          docker run --rm --network host --entrypoint sh minio/mc:latest -c \
+            "mc alias set local http://localhost:9000 minioadmin minioadmin && mc mb -p local/compass-data"
+      # NON-TEST compile of the cloud feature: no other job builds this
+      # combination (clippy/msrv build without the feature; tests build with
+      # cfg(test)), so a cfg gate that hides an item from the release cloud
+      # build otherwise sails through green checks and breaks docker builds.
+      - run: cargo check -p compass --features object-storage
+      # The s3_integration tests skip silently without the env; guard against
+      # env-name drift turning this job into a green no-op.
+      - run: |
+          cargo test -p compass --features object-storage -- --nocapture 2>&1 | tee /tmp/cloud-tests.log
+          if grep -q '^skipped: COMPASS_TEST_S3_BUCKET' /tmp/cloud-tests.log; then
+            echo '::error::s3_integration tests were skipped — MinIO env wiring is broken'
+            exit 1
+          fi
+
+  # Developer Certificate of Origin: every PR commit carries a Signed-off-by
+  # trailer. Dependency-free check over the PR range.
+  dco:
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - run: |
+          missing=0
+          for sha in $(git rev-list --no-merges ${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }}); do
+            if ! git log -1 --format=%B "$sha" | grep -q '^Signed-off-by: '; then
+              echo "::error::commit $sha is missing a Signed-off-by trailer (git commit -s)"
+              missing=1
+            fi
+          done
+          exit $missing
+
   msrv:
     runs-on: ubuntu-24.04
     steps:

ARCHITECTURE.md

@@ -67,18 +67,22 @@ Per-collection state lives under `$DATA_DIR/<collection>/`:
 
 ```
 data/<collection>/
-  meta.json                 CollectionMetadata (name, default vector space, vector_spaces map)
-  chunks.bin                Append-only log of Chunk records
-  metadata.bin              Per-chunk metadata (typed values, bitset-faceted)
-  fts/                      Tantivy directory
-  vectors/<space>/
-    index.usearch           USearch HNSW (CPU) — mmap-backed
-    index.cuvs              cuVS HNSW (GPU build) — when COMPASS_BACKEND=gpu
-    index.keymap            Internal HNSW key -> external chunk id mapping
-    vectors.bin             Raw float buffer (used for brute-force fallback + rebuilds)
+  collection.json           Collection metadata (name, config, vector_spaces map, applied_seq)
+  chunks.redb               Chunk bodies + metadata (redb; disk source of truth)
+  relations.redb            Typed many-to-many chunk relations (redb)
   relationships.bin         Parent-child + sibling edges
+  tantivy/                  Tantivy FTS index directory
+  vectors/
+    <space>.index           USearch HNSW graph — mmap-backed
+    <space>.keymap          Internal HNSW key -> external chunk id mapping
+    <space>.bin             CMV2 mmap vector file (torn-append-safe, per-batch durable)
 ```
 
+In cloud mode the object-storage bucket additionally holds, per collection:
+`collection.json` (bucket config), `manifest` (LSM manifest, CAS-committed),
+`wal/{uuid}.frag` (WAL fragments), `segments/{uuid}` (CSEG0002 sectioned
+segments), and `id-alloc` (CAS-leased chunk-id blocks).
+
 The disk format is the contract. Bumping it requires a migration path documented in CHANGELOG.md.
 
 ## Rebuild flow (model upgrades)

CHANGELOG.md

@@ -4,6 +4,33 @@ All notable changes to this project are documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+
+### Added — "warm serverless"
+
+- **Stateless writer role** (`COMPASS_ROLE=writer`): durable-append-only nodes with no local indexes and instant boot. Writes validate against the bucket's collection config, mint ids from CAS-leased blocks, append one WAL fragment, and return its `seq`. Reads and delete-by-filter are refused with clear errors. Consistency contract: durable immediately, searchable on serving nodes within the refresh interval.
+- **Id-block allocator** (`{ns}/id-alloc`): in cloud mode every ingest path claims id blocks via CAS, so attached nodes and stateless writers can never mint colliding ids. Pre-v0.4 namespaces migrate automatically (seeded from the bucket-derived high-water mark). Do not run v0.3 and v0.4 writers against one bucket during a rolling upgrade.
+- **Bucket collection config** (`{ns}/collection.json`): vector-space specs, default space, `created_at`, and `CollectionConfig` are durable in the bucket and survive cold rebuilds (previously specs were re-inferred as `model:"recovered"` and `embed_model` was silently lost). Vector-space CRUD is bucket-first CAS; zero-ingest collections are discoverable from a fresh disk.
+- **Manifest refresh + read-your-writes**: serving nodes converge with other nodes' writes via a background refresher (`COMPASS_REFRESH_INTERVAL`, default 5s) using a per-collection seq tracker that never double-applies a node's own fragments. Config changes sync on refresh; a deleted collection detaches; a recreated one re-attaches. Write responses carry `seq`; `SearchRequest.min_seq` refreshes-then-serves with a bounded wait.
+- **Lazy attach + LRU detach** (`COMPASS_LAZY_ATTACH`, `COMPASS_MAX_ATTACHED`): boot registers bucket namespaces and attaches on first request (stampede-safe, one rebuild); past the budget the least-recently-used collection detaches and re-attaches on demand — the bucket is the source of truth. Default off; local mode unchanged.
+- **CI**: object-storage build + real-S3 integration tests run against MinIO on every PR (with a silent-skip guard); DCO sign-off enforced on PR commits (merge commits exempt).
+
+### Fixed
+
+- A completed vector-space rebuild (`POST .../rebuild`) never activated: the space stayed `status="building"` and the rebuilt index was not served until restart. Rebuild completion now flips the persisted status (CAS in cloud mode) and hot-loads the index; activation failure is reported as a failed rebuild.
+- Searching or ingesting into a missing collection returned HTTP 500/400; typed not-found errors now map to 404 across all endpoints (replacing three copies of substring-based status sniffing).
+- Facet counts were wiped by every ingest after the first (each batch replaced the accumulated facet state; latent since v0.2), came back empty after any restart (nothing rebuilt them from disk), and counted deleted chunks until a full FTS rebuild. Facets are now roaring treemaps keyed by chunk id: batches accumulate, the load/rebuild scan reconstructs them, and counts intersect the live-id universe so tombstoned chunks are excluded. Found by the new live-stack E2E harness (`scripts/e2e.sh`, 44 checks across every endpoint and both node roles).
+- Warm restarts of an actively-written collection were O(collection size): batched HNSW persistence legitimately leaves the index file behind the mmap, and the load path treated that as corruption and re-inserted every vector (20.2s vs v0.3.0's 1.1s at 100k chunks in the comparison bench). Load now heals incrementally — append only the missing tail rows from the mmap, save, and serve mmap-backed. Warm restart at 100k: 1.6s.
+- Sub-1000-vector collections never persisted the vector keymap, silently relying on identity key→id mapping that returned wrong chunk ids once ids were non-dense (exposed by block allocation; latent since v0.2). The keymap is now saved on every build and synthesized as identity for pre-fix directories.
+
+### Changed
+
+- Pork audit (three independent review passes): −1,200 lines of dead weight removed — the unwired VectorIndex/GPU backend plumbing (`COMPASS_BACKEND` did nothing), a third never-called filter evaluator, never-wired filter-index persistence codecs, the legacy vector writer, the `rayon` dependency, and assorted dead fields/params. `delete_by_filter` now resolves ids through the same roaring filter-index pushdown as search (one filter semantics, not three). The `dead_code` lint is enabled again crate-wide. `collections/mod.rs` shrank from 7,100 to 3,700 lines (test modules extracted to files).
+
+### Scope & limitations (honest)
+
+- Warm, not cold: attach cost is proportional to collection size until the sectioned segment format + serve-from-storage indexes land (roadmap Phases 5–6). Cross-node convergence is periodic (refresh interval), not synchronous — use `min_seq` when you need read-your-writes.
+
 ## [0.3.0] - 2026-07-03
 
 ### Added

CLAUDE.md

@@ -26,7 +26,7 @@ docker run -p 4001:4001 -v ./data:/app/data compass
 crates/
   compass/                Main engine binary (Axum API, search, scoring, embed)
   compass-index-api/      VectorIndex trait (no I/O, no async)
-  compass-vector-gpu/     Optional cuVS GPU backend (--features gpu, Linux + CUDA)
+  compass-vector-gpu/     cuVS GPU backend crate (standalone; not yet wired into the engine)
 ```
 
 ## Architecture
@@ -100,7 +100,10 @@ POST   /collections/:name/vector-spaces/:space/rebuild Trigger re-embedding
 GET    /collections/:name/vector-spaces/:space/status  Rebuild progress
 PUT    /collections/:name/default-vector-space         Switch default space
 
+GET    /collections/:name/segments/at                  Temporal segment lookup (TAMS)
+
 GET    /health                                         Health check
+GET    /metrics                                        Prometheus-text metrics
 ```
 
 ## Embedding Models

Cargo.toml

@@ -18,6 +18,7 @@ rust-version = "1.88"
 [workspace.dependencies]
 # Async runtime + web framework
 axum = { version = "0.8", features = ["json"] }
+tower = { version = "0.5", features = ["limit"] }
 tokio = { version = "1", features = ["full"] }
 tower-http = { version = "0.6", features = ["cors"] }
 

README.md

@@ -566,7 +566,10 @@ POST   /collections/:name/vector-spaces/:space/rebuild Trigger re-embedding
 GET    /collections/:name/vector-spaces/:space/status  Rebuild progress
 PUT    /collections/:name/default-vector-space         Switch default space
 
+GET    /collections/:name/segments/at                  Temporal segment lookup (TAMS)
+
 GET    /health                                         Health check
+GET    /metrics                                        Prometheus-text metrics
 ```
 
 ## Contributing

crates/compass/Cargo.toml

@@ -14,26 +14,23 @@ path = "src/main.rs"
 
 [features]
 default = []
-# Opt-in GPU vector backend via the compass-vector-gpu crate.
 # Requires CUDA 12+ and a Linux host. See ARCHITECTURE.md for build details.
-gpu = ["dep:compass-vector-gpu"]
 # Opt-in object-storage backend (S3/GCS/Azure/MinIO) via the object_store crate.
 # Off by default — local-first deployments pull in zero extra dependencies.
 object-storage = ["dep:object_store", "dep:futures"]
 
 [dependencies]
 # Internal trait crate — defines VectorIndex, IndexParams, IndexError.
 # Stable surface that pluggable backends bind to.
-compass-index-api = { workspace = true }
 
 axum = { workspace = true }
+tower = { workspace = true }
 tokio = { workspace = true }
 tower-http = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 chrono = { workspace = true }
 uuid = { workspace = true }
-rayon = { workspace = true }
 half = { workspace = true }
 tantivy = { workspace = true }
 usearch = { workspace = true }
@@ -58,5 +55,3 @@ reqwest = { workspace = true }
 tracing = { workspace = true }
 tracing-subscriber = { workspace = true }
 
-# Optional GPU backend, enabled with --features gpu
-compass-vector-gpu = { path = "../compass-vector-gpu", optional = true }

crates/compass/src/api/collections.rs

@@ -16,7 +16,7 @@ pub async fn create_collection(
         .manager
         .create_collection(&req.name, req.vector_spaces, req.embedding_dims, req.config)
         .await
-        .map_err(|e| (StatusCode::BAD_REQUEST, e.to_string()))?;
+        .map_err(|e| crate::api::error_response(e, StatusCode::BAD_REQUEST))?;
 
     Ok((StatusCode::CREATED, Json(collection_to_info(&collection))))
 }
@@ -66,7 +66,7 @@ pub async fn add_vector_space(
         .manager
         .add_vector_space(&name, &req.name, req.dims, &req.model)
         .await
-        .map_err(|e| (StatusCode::BAD_REQUEST, e.to_string()))?;
+        .map_err(|e| crate::api::error_response(e, StatusCode::BAD_REQUEST))?;
 
     Ok((
         StatusCode::CREATED,
@@ -114,7 +114,7 @@ pub async fn delete_vector_space(
         .manager
         .delete_vector_space(&name, &space)
         .await
-        .map_err(|e| (StatusCode::BAD_REQUEST, e.to_string()))?;
+        .map_err(|e| crate::api::error_response(e, StatusCode::BAD_REQUEST))?;
     Ok(StatusCode::NO_CONTENT)
 }
 
@@ -128,7 +128,7 @@ pub async fn set_default_vector_space(
         .manager
         .set_default_vector_space(&name, &req.name)
         .await
-        .map_err(|e| (StatusCode::BAD_REQUEST, e.to_string()))?;
+        .map_err(|e| crate::api::error_response(e, StatusCode::BAD_REQUEST))?;
     Ok(StatusCode::OK)
 }
 
@@ -162,7 +162,7 @@ pub async fn trigger_rebuild(
         .manager
         .get_all_chunk_data(&name)
         .await
-        .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?;
+        .map_err(|e| crate::api::error_response(e, StatusCode::INTERNAL_SERVER_ERROR))?;
 
     let vectors_dir = state.manager.vectors_dir(&name);
 
@@ -177,6 +177,7 @@ pub async fn trigger_rebuild(
         req.batch_size,
         state.manager.rebuild_tracker.clone(),
         name,
+        state.manager.clone(),
     )
     .await
     .map_err(|e| (StatusCode::CONFLICT, e))?;

crates/compass/src/api/delete.rs

@@ -16,26 +16,15 @@ use axum::Json;
 use std::sync::Arc;
 
 fn map_err(e: Box<dyn std::error::Error + Send + Sync>) -> (StatusCode, String) {
-    let msg = e.to_string();
-    if msg.contains("not found") {
-        (StatusCode::NOT_FOUND, msg)
-    } else {
-        // Log the detail server-side; internal errors (paths, backends, redb
-        // internals) don't belong in response bodies.
-        tracing::error!("delete handler error: {msg}");
-        (
-            StatusCode::INTERNAL_SERVER_ERROR,
-            "internal error (see server logs)".to_string(),
-        )
-    }
+    crate::api::error_response(e, StatusCode::INTERNAL_SERVER_ERROR)
 }
 
 /// DELETE /collections/:name/chunks/:id
 pub async fn delete_chunk(
     State(state): State<Arc<AppState>>,
     Path((name, id)): Path<(String, u64)>,
 ) -> Result<Json<DeleteResponse>, (StatusCode, String)> {
-    let deleted = state
+    let (deleted, seq) = state
         .manager
         .delete_chunks(&name, &[id])
         .await
@@ -47,7 +36,7 @@ pub async fn delete_chunk(
             format!("chunk {id} not found or already deleted"),
         ));
     }
-    Ok(Json(DeleteResponse { deleted }))
+    Ok(Json(DeleteResponse { deleted, seq }))
 }
 
 /// POST /collections/:name/compact — fold S3 segments + WAL into one segment,
@@ -78,18 +67,24 @@ pub async fn delete_by_query(
     }
 
     let mut deleted = 0usize;
+    let mut seq: Option<u64> = None;
     if !req.ids.is_empty() {
-        deleted += state
+        let (n, s) = state
             .manager
             .delete_chunks(&name, &req.ids)
             .await
             .map_err(map_err)?;
+        deleted += n;
+        seq = s.or(seq);
     }
     if !req.filters.is_empty() {
         // If the filter-delete fails after an ids-delete succeeded, report the
         // partial progress — deletes already applied are not undone.
         match state.manager.delete_by_filter(&name, &req.filters).await {
-            Ok(n) => deleted += n,
+            Ok((n, s)) => {
+                deleted += n;
+                seq = s.or(seq);
+            }
             Err(e) => {
                 let (code, msg) = map_err(e);
                 return Err((
@@ -101,5 +96,5 @@ pub async fn delete_by_query(
             }
         }
     }
-    Ok(Json(DeleteResponse { deleted }))
+    Ok(Json(DeleteResponse { deleted, seq }))
 }