diff --git a/.github/workflows/base.yml b/.github/workflows/base.yml index c71506d43..cc0b1cb3b 100644 --- a/.github/workflows/base.yml +++ b/.github/workflows/base.yml @@ -261,6 +261,7 @@ jobs: sccache --start-server || true cargo test --manifest-path common/Cargo.toml --all-features --no-fail-fast -- --nocapture cargo test --manifest-path jupiter/Cargo.toml --all-features --no-fail-fast -- --nocapture + cargo test --manifest-path jupiter-migrate/Cargo.toml --all-features --no-fail-fast -- --nocapture cargo test --manifest-path ceres/Cargo.toml --all-features --no-fail-fast -- --nocapture cargo test --manifest-path vault/Cargo.toml --all-features --no-fail-fast -- --nocapture cargo test --manifest-path saturn/Cargo.toml --all-features --no-fail-fast -- --nocapture @@ -269,3 +270,47 @@ jobs: # Note: The fuse/scorpio job has been removed as scorpio has been moved # to its own repository: https://github.com/gitmono-dev/scorpiofs + + snapshot-contract: + name: Snapshot PostgreSQL and index gates + runs-on: ubuntu-latest + timeout-minutes: 45 + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: snapshot_fixture + POSTGRES_PASSWORD: fixture-local-only + POSTGRES_DB: snapshot_test + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U snapshot_fixture -d snapshot_test" + --health-interval 5s --health-timeout 5s --health-retries 20 + env: + CARGO_TERM_COLOR: always + RUSTUP_TOOLCHAIN: stable + MEGA_SNAPSHOT_TEST_DATABASE_URL: postgres://snapshot_fixture:fixture-local-only@127.0.0.1:5432/snapshot_test + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - name: Install focused Rust build dependencies + run: | + sudo apt-get update + sudo apt-get install -y cmake build-essential pkg-config libssl-dev nettle-dev protobuf-compiler + rustup toolchain install stable --profile minimal + - uses: Swatinem/rust-cache@v2 + with: + shared-key: base-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }} + cache-on-failure: true + - name: Snapshot migration and UTC upgrade tests + run: cargo test -p jupiter-migrate --lib snapshot --locked -j 2 -- --include-ignored --nocapture + - name: Source identity and immutable node storage tests + run: | + cargo test -p jupiter --lib snapshot_storage --locked -j 2 + cargo test -p jupiter --lib namespace_storage --locked -j 2 + - name: Snapshot contracts, PostgreSQL transactions and million-binding gate + run: cargo test -p ceres --lib snapshot --locked -j 2 -- --include-ignored --nocapture + - name: Publication receipts, ref CAS, rollback and concurrency on both databases + run: cargo test -p jupiter --lib publication_storage --locked -j 2 -- --include-ignored --nocapture diff --git a/Cargo.lock b/Cargo.lock index ea21f9567..506d82b90 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4294,6 +4294,7 @@ dependencies = [ "tempfile", "tokio", "tracing", + "url", ] [[package]] diff --git a/ceres/Cargo.toml b/ceres/Cargo.toml index b7a9c7209..6709d0fdb 100644 --- a/ceres/Cargo.toml +++ b/ceres/Cargo.toml @@ -29,7 +29,7 @@ rand = { workspace = true, features = ["thread_rng"] } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } sha1 = { workspace = true } -sha2 = { workspace = true, optional = true } +sha2 = { workspace = true } tempfile = { workspace = true, optional = true } thiserror = { workspace = true, optional = true } tracing = { workspace = true } @@ -46,7 +46,7 @@ tokio-util = { workspace = true } rkyv = { workspace = true } [features] -fastcdc = ["dep:sha2", "dep:tempfile", "dep:thiserror"] +fastcdc = ["dep:tempfile", "dep:thiserror"] migrate = ["jupiter/migrate"] [dev-dependencies] diff --git a/ceres/src/application/api_service/import_api_service.rs b/ceres/src/application/api_service/import_api_service.rs index 3a3adee37..a1f7e7481 100644 --- a/ceres/src/application/api_service/import_api_service.rs +++ b/ceres/src/application/api_service/import_api_service.rs @@ -112,32 +112,26 @@ impl ApiHandler for ImportApiService { async fn get_root_commit(&self) -> Result { let storage = self.storage.git_db_storage(); - let refs = storage.get_default_ref(self.repo.repo_id).await?.unwrap(); - self.get_commit_by_hash(&refs.ref_git_id).await + let source = crate::application::snapshot::source::resolve_import_commit( + &storage, + self.repo.repo_id, + None, + ) + .await?; + self.get_commit_by_hash(&source.commit_oid).await } - /// Note: The `refs` parameter is intentionally ignored for import repositories, - /// as they do not support selecting refs. The default ref is always used. - async fn get_root_tree(&self, _: Option<&str>) -> Result { + /// Resolve once, then read that immutable root. An explicit revision must + /// never silently fall back to the repository's current default branch. + async fn get_root_tree(&self, refs: Option<&str>) -> Result { let storage = self.storage.git_db_storage(); - let refs = storage - .get_default_ref(self.repo.repo_id) - .await - .unwrap() - .unwrap(); - - let root_commit = storage - .get_commit_by_hash(self.repo.repo_id, &refs.ref_git_id) - .await - .unwrap() - .unwrap(); - Ok(Tree::from_git_model( - storage - .get_tree_by_hash(self.repo.repo_id, &root_commit.tree) - .await - .unwrap() - .unwrap(), - )) + let source = crate::application::snapshot::source::resolve_import_commit( + &storage, + self.repo.repo_id, + refs, + ) + .await?; + crate::application::snapshot::source::read_import_root(&storage, &source).await } async fn get_tree_by_hash(&self, hash: &str) -> Result { @@ -146,7 +140,7 @@ impl ApiHandler for ImportApiService { .git_db_storage() .get_tree_by_hash(self.repo.repo_id, hash) .await? - .unwrap(); + .ok_or_else(|| MegaError::NotFound(format!("import tree not found: {hash}")))?; Ok(Tree::from_git_model(model)) } @@ -155,7 +149,7 @@ impl ApiHandler for ImportApiService { let commit = storage .get_commit_by_hash(self.repo.repo_id, hash) .await? - .unwrap(); + .ok_or_else(|| MegaError::NotFound(format!("import commit not found: {hash}")))?; Ok(Commit::from_git_model(commit)) } diff --git a/ceres/src/application/mod.rs b/ceres/src/application/mod.rs index 0f27558ec..0e5e9ab8e 100644 --- a/ceres/src/application/mod.rs +++ b/ceres/src/application/mod.rs @@ -5,4 +5,5 @@ pub mod build_trigger; pub mod code_edit; pub mod member_identity; pub mod notification; +pub mod snapshot; pub mod webhook; diff --git a/ceres/src/application/snapshot/catalog.rs b/ceres/src/application/snapshot/catalog.rs new file mode 100644 index 000000000..b112ed851 --- /dev/null +++ b/ceres/src/application/snapshot/catalog.rs @@ -0,0 +1,395 @@ +//! Source resolution and fixed-tree membership, independent of live HTTP path +//! routing. This is an internal metadata boundary, NOT authorization, retention, +//! or a namespace publisher. A public read service must enforce those gates. + +use common::errors::MegaError; +use jupiter::storage::{ + Storage, + base_storage::{BaseStorage, StorageConnector}, + git_db_storage::GitDbStorage, + mono_storage::MonoStorage, + snapshot_storage::{ScopeAttestation, ScopeProofKind, SnapshotStorage, SourceKind}, +}; + +use super::{ + object::{self, EntryKind, FixedEntry, ObjectKind}, + source::resolve_import_commit, +}; +use crate::model::snapshot::{ + ObjectFormat, ObjectId, RelativePath, RepoPath, SourceId, SourceSelector, SourceSnapshot, +}; + +const MAX_TREE_BYTES: usize = 16 * 1024 * 1024; + +#[derive(Clone)] +pub struct SourceCatalog { + proofs: SnapshotStorage, + mono: MonoStorage, + imports: GitDbStorage, +} + +impl SourceCatalog { + pub fn new(storage: &Storage) -> Self { + Self::from_base(storage.mono_storage().base.clone()) + } + + fn from_base(base: BaseStorage) -> Self { + Self { + proofs: SnapshotStorage { base: base.clone() }, + mono: MonoStorage { base: base.clone() }, + imports: GitDbStorage { base }, + } + } + + pub async fn register_native(&self) -> Result { + let source = self.proofs.ensure_source(SourceKind::Native, 0).await?; + SourceId::new(source.source_id).map_err(invalid_stored) + } + + /// Initial lookup only. Once resolved, reads use the persisted backend ID, + /// never this current registry path. Registration alone publishes no view. + pub async fn register_import(&self, path: &RepoPath) -> Result { + let repo = self + .imports + .find_git_repo_exact_match(path.as_str()) + .await? + .ok_or_else(|| MegaError::NotFound("import repository not found".into()))?; + let source = self + .proofs + .ensure_source(SourceKind::Import, repo.id) + .await?; + SourceId::new(source.source_id).map_err(invalid_stored) + } + + pub async fn resolve(&self, selector: &SourceSelector) -> Result { + let (source_id, scope) = match selector { + SourceSelector::SourceCommit { + source_id, + scope_path, + .. + } + | SourceSelector::SourceRef { + source_id, + scope_path, + .. + } => (source_id, scope_path), + }; + let backend = self.backend(source_id).await?; + + // A recorded proof is authoritative even after ref/registry removal. + if let SourceSelector::SourceCommit { commit_oid, .. } = selector + && let Some(proof) = self + .proofs + .scope(source_id.as_str(), scope.as_str(), commit_oid.as_str()) + .await? + { + let source = descriptor( + source_id.clone(), + scope.clone(), + commit_oid.as_str(), + &proof.root_tree_oid, + )?; + self.tree(&backend, &source.root_tree_oid).await?; + return Ok(source); + } + + let (commit, tree, proof_kind) = match backend.kind.as_str() { + "import" => { + // A new observation must use the actual current import root. + // Multi-source atomicity requires the namespace publisher, not + // this individually reproducible source observation. + let repo = self + .imports + .find_git_repo_by_id(backend.repo_id) + .await? + .ok_or_else(|| { + MegaError::NotFound("import registry entry no longer exists".into()) + })?; + if repo.repo_path != scope.as_str() { + return Err(MegaError::bad_request( + "unattested import scope is not its registered root", + )); + } + let revision = match selector { + SourceSelector::SourceCommit { commit_oid, .. } => commit_oid.as_str(), + SourceSelector::SourceRef { ref_name, .. } => ref_name.as_str(), + }; + let resolved = + resolve_import_commit(&self.imports, backend.repo_id, Some(revision)).await?; + ( + resolved.commit_oid, + resolved.root_tree_oid, + ScopeProofKind::ImportCommit, + ) + } + "native" => { + let SourceSelector::SourceRef { ref_name, .. } = selector else { + return Err(MegaError::bad_request( + "SCOPE_UNKNOWN: native commit has no proof for this scope", + )); + }; + // Ref row captures commit + scope together. Do not accept a + // commit from the global table and guess that it is a root. + let selected = self + .mono + .get_ref_at_path(scope.as_str(), ref_name.as_str()) + .await? + .ok_or_else(|| MegaError::NotFound("native scoped ref not found".into()))?; + let commit = self + .mono + .get_commit_by_hash(&selected.ref_commit_hash) + .await? + .ok_or_else(|| MegaError::NotFound("native commit not found".into()))?; + if selected.ref_tree_hash != commit.tree { + return Err(MegaError::Unavailable( + "native ref and commit root tree disagree".into(), + )); + } + ( + commit.commit_id, + commit.tree, + ScopeProofKind::NativeRefObserved, + ) + } + _ => unreachable!("backend validates kind"), + }; + let source = descriptor(source_id.clone(), scope.clone(), &commit, &tree)?; + self.tree(&backend, &source.root_tree_oid).await?; + self.record(&source, proof_kind, None).await?; + Ok(source) + } + + /// Derive a native child from one already-attested immutable root. The + /// child retains the root commit's provenance, not a separately moving ref. + pub async fn project_native( + &self, + base: &SourceSnapshot, + scope: &RepoPath, + ) -> Result { + let backend = self.validate(base).await?; + if backend.kind != "native" { + return Err(MegaError::bad_request( + "native projection requires a native source", + )); + } + let path = scope + .relative_to(&base.scope_path) + .ok_or_else(|| MegaError::bad_request("projection is outside attested scope"))?; + let entry = self.locate_in(&backend, base, &path).await?; + if entry.kind != EntryKind::Directory { + return Err(MegaError::bad_request( + "projection target is not a directory", + )); + } + self.tree(&backend, &entry.oid).await?; + let source = SourceSnapshot { + scope_path: scope.clone(), + root_tree_oid: entry.oid, + ..base.clone() + }; + self.record( + &source, + ScopeProofKind::NativeScopeProjection, + Some(base.commit_oid.to_string()), + ) + .await?; + Ok(source) + } + + /// Verify the descriptor itself before object access; a caller-supplied + /// valid-looking root OID is not proof. Does not consult current refs. + async fn validate( + &self, + source: &SourceSnapshot, + ) -> Result { + let backend = self.backend(&source.source_id).await?; + let proof = self + .proofs + .scope( + source.source_id.as_str(), + source.scope_path.as_str(), + source.commit_oid.as_str(), + ) + .await? + .ok_or_else(|| { + MegaError::bad_request("SCOPE_UNKNOWN: source descriptor is not attested") + })?; + if proof.root_tree_oid != source.root_tree_oid.as_str() { + return Err(MegaError::bad_request( + "source root does not match attestation", + )); + } + Ok(backend) + } + + pub async fn locate( + &self, + source: &SourceSnapshot, + path: &RelativePath, + ) -> Result { + let backend = self.validate(source).await?; + self.locate_in(&backend, source, path).await + } + + /// Prove the exact (source, path, kind, OID) before a physical CAS fetch. + /// This is object membership only; the read facade must also check current + /// source/path authorization and an active retention lease on every read. + pub async fn prove_object( + &self, + source: &SourceSnapshot, + path: &RelativePath, + kind: ObjectKind, + oid: &ObjectId, + ) -> Result<(), MegaError> { + let entry = self.locate(source, path).await?; + if entry.kind.object_kind()? != kind || &entry.oid != oid { + return Err(MegaError::bad_request( + "object is not at this fixed-source path", + )); + } + Ok(()) + } + + pub async fn read_tree_payload( + &self, + source: &SourceSnapshot, + path: &RelativePath, + oid: &ObjectId, + ) -> Result, MegaError> { + let backend = self.validate(source).await?; + let entry = self.locate_in(&backend, source, path).await?; + if entry.kind != EntryKind::Directory || &entry.oid != oid { + return Err(MegaError::bad_request( + "tree is not at this fixed-source path", + )); + } + self.tree_payload(&backend, oid).await + } + + async fn locate_in( + &self, + backend: &callisto::snapshot_source::Model, + source: &SourceSnapshot, + path: &RelativePath, + ) -> Result { + let mut entry = FixedEntry { + name: String::new(), + kind: EntryKind::Directory, + oid: source.root_tree_oid.clone(), + }; + if path.as_str().is_empty() { + return Ok(entry); + } + for component in path.as_str().split('/') { + if entry.kind != EntryKind::Directory { + return Err(MegaError::bad_request( + "snapshot traversal target is not a directory", + )); + } + entry = self + .tree(backend, &entry.oid) + .await? + .into_iter() + .find(|entry| entry.name == component) + .ok_or_else(|| MegaError::NotFound("path not found in fixed source".into()))?; + } + Ok(entry) + } + + async fn backend( + &self, + source: &SourceId, + ) -> Result { + let backend = self + .proofs + .source(source.as_str()) + .await? + .ok_or_else(|| MegaError::NotFound("snapshot source not found".into()))?; + if !((backend.kind == "native" && backend.repo_id == 0) + || (backend.kind == "import" && backend.repo_id > 0)) + { + return Err(MegaError::Unavailable( + "invalid stored snapshot backend".into(), + )); + } + Ok(backend) + } + + async fn tree( + &self, + backend: &callisto::snapshot_source::Model, + oid: &ObjectId, + ) -> Result, MegaError> { + object::decode_tree(&self.tree_payload(backend, oid).await?) + } + + async fn tree_payload( + &self, + backend: &callisto::snapshot_source::Model, + oid: &ObjectId, + ) -> Result, MegaError> { + let payload = if backend.kind == "native" { + self.mono + .get_tree_by_hash(oid.as_str()) + .await? + .map(|tree| tree.sub_trees) + } else { + self.imports + .get_tree_by_hash(backend.repo_id, oid.as_str()) + .await? + .map(|tree| tree.sub_trees) + } + .ok_or_else(|| MegaError::NotFound("fixed-source tree is missing".into()))?; + // DB-backed trees are already materialized by the driver. This bound + // protects decoding, not the database allocation. Blob streaming is a + // separate read boundary and must enforce its bound before collecting. + if payload.len() > MAX_TREE_BYTES { + return Err(MegaError::bad_request("snapshot tree exceeds byte limit")); + } + object::verify_object(ObjectKind::Tree, oid, &payload)?; + Ok(payload) + } + + async fn record( + &self, + source: &SourceSnapshot, + kind: ScopeProofKind, + proof_oid: Option, + ) -> Result<(), MegaError> { + self.proofs + .record_scope_in( + self.proofs.base.get_connection(), + &ScopeAttestation { + source_id: source.source_id.to_string(), + scope_path: source.scope_path.to_string(), + commit_oid: source.commit_oid.to_string(), + root_tree_oid: source.root_tree_oid.to_string(), + proof_kind: kind, + proof_oid, + }, + ) + .await + } +} + +fn invalid_stored(error: impl std::fmt::Display) -> MegaError { + MegaError::Unavailable(format!("invalid stored source identity: {error}")) +} + +fn descriptor( + source_id: SourceId, + scope_path: RepoPath, + commit: &str, + tree: &str, +) -> Result { + Ok(SourceSnapshot { + source_id, + scope_path, + object_format: ObjectFormat::Sha1, + commit_oid: ObjectId::new(commit).map_err(invalid_stored)?, + root_tree_oid: ObjectId::new(tree).map_err(invalid_stored)?, + }) +} + +#[cfg(test)] +mod tests; diff --git a/ceres/src/application/snapshot/catalog/tests.rs b/ceres/src/application/snapshot/catalog/tests.rs new file mode 100644 index 000000000..5b8fd819c --- /dev/null +++ b/ceres/src/application/snapshot/catalog/tests.rs @@ -0,0 +1,451 @@ +use std::sync::Arc; + +use callisto::{ + git_commit, git_repo, git_tree, import_refs, mega_refs, mega_tree, + sea_orm_active_enums::RefTypeEnum, +}; +use git_internal::internal::{ + metadata::EntryMeta, + object::{ + blob::Blob, + commit::Commit, + tree::{Tree, TreeItem, TreeItemMode}, + }, +}; +use jupiter::utils::converter::IntoGitModel; +use sea_orm::{ColumnTrait, EntityTrait, IntoActiveModel, QueryFilter, sea_query::Expr}; +use tempfile::TempDir; + +use super::*; + +async fn fixture() -> (TempDir, SourceCatalog) { + let dir = TempDir::new().unwrap(); + let connection = jupiter::tests::test_db_connection(dir.path()).await; + jupiter_migrate::apply_migrations(&connection, true) + .await + .unwrap(); + ( + dir, + SourceCatalog::from_base(BaseStorage::new(Arc::new(connection))), + ) +} + +fn path(value: &str) -> RepoPath { + RepoPath::new(value).unwrap() +} +fn relative(value: &str) -> RelativePath { + RelativePath::new(value).unwrap() +} + +fn ref_selector(id: &SourceId, scope: &str) -> SourceSelector { + SourceSelector::SourceRef { + source_id: id.clone(), + scope_path: path(scope), + ref_name: crate::model::snapshot::RefName::new("refs/heads/main").unwrap(), + } +} + +fn fixed(source: &SourceSnapshot) -> SourceSelector { + SourceSelector::SourceCommit { + source_id: source.source_id.clone(), + scope_path: source.scope_path.clone(), + commit_oid: source.commit_oid.clone(), + } +} + +fn file_tree(content: &str) -> Tree { + Tree::from_tree_items(vec![TreeItem { + mode: TreeItemMode::Blob, + name: "file.txt".into(), + id: Blob::from_content(content).id, + }]) + .unwrap() +} + +async fn native_commit(catalog: &SourceCatalog, tree: Tree) -> Commit { + let commit = Commit::from_tree_id(tree.id, vec![], "snapshot fixture"); + catalog + .mono + .save_mega_trees(vec![tree], commit.id, None) + .await + .unwrap(); + catalog + .mono + .save_mega_commits(vec![commit.clone()], None) + .await + .unwrap(); + commit +} + +async fn native_ref(catalog: &SourceCatalog, scope: &str, commit: &Commit) { + let now = chrono::Utc::now().naive_utc(); + catalog + .mono + .save_refs( + mega_refs::Model { + id: common::utils::generate_id(), + path: scope.into(), + ref_name: "refs/heads/main".into(), + ref_commit_hash: commit.id.to_string(), + ref_tree_hash: commit.tree_id.to_string(), + created_at: now, + updated_at: now, + is_cl: false, + }, + None, + ) + .await + .unwrap(); +} + +async fn import_repo(catalog: &SourceCatalog, repo_id: i64, scope: &str) { + let now = chrono::Utc::now().naive_utc(); + catalog + .imports + .save_git_repo(git_repo::Model { + id: repo_id, + repo_path: scope.into(), + repo_name: format!("r{repo_id}"), + created_at: now, + updated_at: now, + }) + .await + .unwrap(); +} + +async fn import_commit(catalog: &SourceCatalog, repo_id: i64, content: &str) -> (Commit, Tree) { + let tree = file_tree(content); + let commit = Commit::from_tree_id(tree.id, vec![], content); + let mut tree_model = tree.clone().into_git_model(EntryMeta::new()); + tree_model.repo_id = repo_id; + git_tree::Entity::insert(tree_model.into_active_model()) + .exec(catalog.imports.get_connection()) + .await + .unwrap(); + let mut commit_model = commit.clone().into_git_model(EntryMeta::new()); + commit_model.repo_id = repo_id; + git_commit::Entity::insert(commit_model.into_active_model()) + .exec(catalog.imports.get_connection()) + .await + .unwrap(); + (commit, tree) +} + +async fn import_ref(catalog: &SourceCatalog, repo_id: i64, commit: &Commit) { + let now = chrono::Utc::now().naive_utc(); + catalog + .imports + .save_ref( + repo_id, + import_refs::Model { + id: common::utils::generate_id(), + repo_id, + ref_name: "refs/heads/main".into(), + ref_git_id: commit.id.to_string(), + ref_type: RefTypeEnum::Branch, + default_branch: true, + created_at: now, + updated_at: now, + }, + ) + .await + .unwrap(); +} + +#[tokio::test] +async fn native_projection_keeps_root_provenance_and_survives_ref_cleanup() { + let (_dir, catalog) = fixture().await; + let child = file_tree("old"); + let child_commit = native_commit(&catalog, child.clone()).await; + let root = Tree::from_tree_items(vec![TreeItem { + mode: TreeItemMode::Tree, + id: child.id, + name: "pkg".into(), + }]) + .unwrap(); + let root_commit = native_commit(&catalog, root).await; + native_ref(&catalog, "/", &root_commit).await; + native_ref(&catalog, "/pkg", &child_commit).await; + let id = catalog.register_native().await.unwrap(); + let pinned = catalog.resolve(&ref_selector(&id, "/")).await.unwrap(); + let projected = catalog + .project_native(&pinned, &path("/pkg")) + .await + .unwrap(); + assert_eq!(projected.commit_oid, pinned.commit_oid); + assert_eq!(projected.root_tree_oid.as_str(), child.id.to_string()); + assert_ne!(projected.commit_oid.as_str(), child_commit.id.to_string()); + let scoped = catalog.resolve(&ref_selector(&id, "/pkg")).await.unwrap(); + assert_eq!(scoped.root_tree_oid, projected.root_tree_oid); + assert_ne!(scoped.id(), projected.id()); + + mega_refs::Entity::delete_many() + .exec(catalog.mono.get_connection()) + .await + .unwrap(); + let reloaded = SourceCatalog::from_base(catalog.mono.base.clone()); + assert_eq!( + reloaded.resolve(&fixed(&projected)).await.unwrap(), + projected + ); + let file = reloaded + .locate(&projected, &relative("file.txt")) + .await + .unwrap(); + assert_eq!(file.oid.as_str(), child.tree_items[0].id.to_string()); + assert!(matches!( + reloaded.locate(&projected, &relative("pkg/file.txt")).await, + Err(MegaError::NotFound(_)) + )); + assert!( + reloaded + .project_native(&projected, &path("/pkgs")) + .await + .is_err() + ); +} + +#[tokio::test] +async fn native_commit_cannot_be_reinterpreted_as_an_unproven_scope() { + let (_dir, catalog) = fixture().await; + let child_commit = native_commit(&catalog, file_tree("scoped")).await; + native_ref(&catalog, "/project", &child_commit).await; + let id = catalog.register_native().await.unwrap(); + let pinned = catalog + .resolve(&ref_selector(&id, "/project")) + .await + .unwrap(); + let wrong = SourceSelector::SourceCommit { + source_id: id, + scope_path: path("/"), + commit_oid: pinned.commit_oid.clone(), + }; + assert!( + matches!(catalog.resolve(&wrong).await, Err(MegaError::BadRequest(message)) if message.contains("SCOPE_UNKNOWN")) + ); + let forged = SourceSnapshot { + scope_path: path("/"), + ..pinned + }; + assert!( + catalog + .locate(&forged, &relative("file.txt")) + .await + .is_err() + ); +} + +#[tokio::test] +async fn import_old_source_survives_move_delete_and_path_reuse() { + let (_dir, catalog) = fixture().await; + import_repo(&catalog, 101, "/third-party/r").await; + let (a, a_tree) = import_commit(&catalog, 101, "A").await; + let (b, _) = import_commit(&catalog, 101, "B").await; + import_ref(&catalog, 101, &a).await; + let id = catalog + .register_import(&path("/third-party/r")) + .await + .unwrap(); + let old = catalog + .resolve(&ref_selector(&id, "/third-party/r")) + .await + .unwrap(); + catalog + .imports + .update_ref(101, "refs/heads/main", &b.id.to_string()) + .await + .unwrap(); + git_repo::Entity::update_many() + .col_expr( + git_repo::Column::RepoPath, + Expr::value("/third-party/moved"), + ) + .filter(git_repo::Column::Id.eq(101)) + .exec(catalog.imports.get_connection()) + .await + .unwrap(); + let new = catalog + .resolve(&ref_selector(&id, "/third-party/moved")) + .await + .unwrap(); + assert_eq!(new.source_id, old.source_id); + assert_ne!(new.commit_oid, old.commit_oid); + assert_eq!(catalog.resolve(&fixed(&old)).await.unwrap(), old); + assert!( + catalog + .resolve(&ref_selector(&id, "/third-party/r")) + .await + .is_err() + ); + + catalog + .imports + .remove_ref(101, "refs/heads/main") + .await + .unwrap(); + git_repo::Entity::delete_by_id(101) + .exec(catalog.imports.get_connection()) + .await + .unwrap(); + import_repo(&catalog, 102, "/third-party/r").await; + let replacement_id = catalog + .register_import(&path("/third-party/r")) + .await + .unwrap(); + assert_ne!(replacement_id, old.source_id); + assert_eq!(catalog.resolve(&fixed(&old)).await.unwrap(), old); + let old_file = catalog.locate(&old, &relative("file.txt")).await.unwrap(); + assert_eq!(old_file.oid.as_str(), a_tree.tree_items[0].id.to_string()); + let forged = SourceSnapshot { + source_id: replacement_id, + ..old + }; + assert!( + catalog + .locate(&forged, &relative("file.txt")) + .await + .is_err() + ); +} + +#[tokio::test] +async fn source_membership_binds_path_kind_and_oid_not_global_cas_presence() { + let (_dir, catalog) = fixture().await; + import_repo(&catalog, 101, "/r").await; + let (a, tree) = import_commit(&catalog, 101, "A").await; + import_ref(&catalog, 101, &a).await; + let id = catalog.register_import(&path("/r")).await.unwrap(); + let source = catalog.resolve(&ref_selector(&id, "/r")).await.unwrap(); + let file_oid = ObjectId::new(tree.tree_items[0].id.to_string()).unwrap(); + catalog + .prove_object(&source, &relative("file.txt"), ObjectKind::Blob, &file_oid) + .await + .unwrap(); + assert!( + catalog + .prove_object(&source, &relative("file.txt"), ObjectKind::Tree, &file_oid) + .await + .is_err() + ); + assert!( + catalog + .prove_object(&source, &relative("missing"), ObjectKind::Blob, &file_oid) + .await + .is_err() + ); + assert!( + catalog + .prove_object( + &source, + &relative("file.txt"), + ObjectKind::Blob, + &source.root_tree_oid + ) + .await + .is_err() + ); + let bytes = catalog + .read_tree_payload(&source, &relative(""), &source.root_tree_oid) + .await + .unwrap(); + object::verify_object(ObjectKind::Tree, &source.root_tree_oid, &bytes).unwrap(); + let foreign = ObjectId::new(Blob::from_content("foreign").id.to_string()).unwrap(); + let forged = SourceSnapshot { + root_tree_oid: foreign, + ..source + }; + assert!(catalog.locate(&forged, &relative("")).await.is_err()); +} + +#[tokio::test] +async fn inconsistent_native_ref_and_corrupt_root_do_not_create_proofs() { + let (_dir, catalog) = fixture().await; + let commit = native_commit(&catalog, file_tree("A")).await; + native_ref(&catalog, "/", &commit).await; + let id = catalog.register_native().await.unwrap(); + mega_refs::Entity::update_many() + .col_expr(mega_refs::Column::RefTreeHash, Expr::value("1".repeat(40))) + .exec(catalog.mono.get_connection()) + .await + .unwrap(); + assert!(matches!( + catalog.resolve(&ref_selector(&id, "/")).await, + Err(MegaError::Unavailable(_)) + )); + mega_refs::Entity::update_many() + .col_expr( + mega_refs::Column::RefTreeHash, + Expr::value(commit.tree_id.to_string()), + ) + .exec(catalog.mono.get_connection()) + .await + .unwrap(); + mega_tree::Entity::update_many() + .col_expr(mega_tree::Column::SubTrees, Expr::value(Vec::::new())) + .exec(catalog.mono.get_connection()) + .await + .unwrap(); + assert!(matches!( + catalog.resolve(&ref_selector(&id, "/")).await, + Err(MegaError::Unavailable(_)) + )); + assert!( + catalog + .proofs + .scope(id.as_str(), "/", &commit.id.to_string()) + .await + .unwrap() + .is_none() + ); +} + +#[tokio::test] +async fn native_symlink_and_gitlink_are_not_traversed_as_directories() { + let (_dir, catalog) = fixture().await; + let tree = Tree::from_tree_items(vec![ + TreeItem { + mode: TreeItemMode::Link, + name: "link".into(), + id: Blob::from_content("../outside").id, + }, + TreeItem { + mode: TreeItemMode::Commit, + name: "submodule".into(), + id: Blob::from_content("commit-shaped fixture").id, + }, + ]) + .unwrap(); + let commit = native_commit(&catalog, tree).await; + native_ref(&catalog, "/", &commit).await; + let id = catalog.register_native().await.unwrap(); + let source = catalog.resolve(&ref_selector(&id, "/")).await.unwrap(); + let link = catalog.locate(&source, &relative("link")).await.unwrap(); + assert_eq!(link.kind, EntryKind::Symlink); + catalog + .prove_object(&source, &relative("link"), ObjectKind::Blob, &link.oid) + .await + .unwrap(); + let sub = catalog + .locate(&source, &relative("submodule")) + .await + .unwrap(); + assert_eq!(sub.kind, EntryKind::Gitlink); + assert!( + catalog + .prove_object(&source, &relative("submodule"), ObjectKind::Blob, &sub.oid) + .await + .is_err() + ); + assert!( + catalog + .locate(&source, &relative("link/child")) + .await + .is_err() + ); + assert!( + catalog + .locate(&source, &relative("submodule/child")) + .await + .is_err() + ); +} diff --git a/ceres/src/application/snapshot/mod.rs b/ceres/src/application/snapshot/mod.rs new file mode 100644 index 000000000..b60cc150e --- /dev/null +++ b/ceres/src/application/snapshot/mod.rs @@ -0,0 +1,9 @@ +//! Immutable source resolution, independent of HTTP and transport implementations. +//! +//! Resolving one source is not a published namespace snapshot: registry history, +//! publication and retention are separate capabilities. + +pub mod catalog; +pub mod object; +pub mod radix; +pub(crate) mod source; diff --git a/ceres/src/application/snapshot/object.rs b/ceres/src/application/snapshot/object.rs new file mode 100644 index 000000000..baf857af9 --- /dev/null +++ b/ceres/src/application/snapshot/object.rs @@ -0,0 +1,146 @@ +//! Strict SHA-1 object decoding for snapshot reads. Do not use the legacy Git +//! parser's thread-local algorithm or non-UTF-8 name conversion on this boundary. + +use std::collections::HashSet; + +use common::errors::MegaError; +use sha1::{Digest, Sha1}; + +use crate::model::snapshot::{MAX_COMPONENT_BYTES, ObjectId}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ObjectKind { + Tree, + Blob, +} + +impl ObjectKind { + pub fn as_str(self) -> &'static str { + match self { + Self::Tree => "tree", + Self::Blob => "blob", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EntryKind { + Directory, + File, + Executable, + Symlink, + Gitlink, +} + +impl EntryKind { + pub fn object_kind(self) -> Result { + match self { + Self::Directory => Ok(ObjectKind::Tree), + Self::File | Self::Executable | Self::Symlink => Ok(ObjectKind::Blob), + Self::Gitlink => Err(MegaError::bad_request( + "snapshot submodule hydration is unsupported", + )), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FixedEntry { + pub name: String, + pub kind: EntryKind, + pub oid: ObjectId, +} + +pub fn verify_object(kind: ObjectKind, oid: &ObjectId, payload: &[u8]) -> Result<(), MegaError> { + let mut hash = Sha1::new(); + hash.update(format!("{} {}\0", kind.as_str(), payload.len()).as_bytes()); + hash.update(payload); + if hex::encode(hash.finalize()) != oid.as_str() { + return Err(MegaError::Unavailable( + "snapshot object integrity failure".into(), + )); + } + Ok(()) +} + +pub fn decode_tree(mut raw: &[u8]) -> Result, MegaError> { + let malformed = || MegaError::Unavailable("malformed or unsupported snapshot tree".into()); + let mut entries = Vec::new(); + let mut names = HashSet::new(); + while !raw.is_empty() { + let space = raw.iter().position(|b| *b == b' ').ok_or_else(malformed)?; + let kind = match &raw[..space] { + b"40000" | b"040000" => EntryKind::Directory, + b"100644" => EntryKind::File, + b"100755" => EntryKind::Executable, + b"120000" => EntryKind::Symlink, + b"160000" => EntryKind::Gitlink, + _ => return Err(malformed()), + }; + raw = &raw[space + 1..]; + let nul = raw.iter().position(|b| *b == 0).ok_or_else(malformed)?; + let name = std::str::from_utf8(&raw[..nul]).map_err(|_| malformed())?; + if name.is_empty() + || name == "." + || name == ".." + || name.contains('/') + || name.len() > MAX_COMPONENT_BYTES + || !names.insert(name.to_owned()) + { + return Err(malformed()); + } + raw = &raw[nul + 1..]; + let oid = raw.get(..20).ok_or_else(malformed)?; + entries.push(FixedEntry { + name: name.to_owned(), + kind, + oid: ObjectId::new(hex::encode(oid)).map_err(|_| malformed())?, + }); + raw = &raw[20..]; + } + entries.sort_by(|a, b| a.name.cmp(&b.name)); + Ok(entries) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn entry(mode: &[u8], name: &[u8]) -> Vec { + [mode, b" ", name, b"\0", &[0x11; 20]].concat() + } + + #[test] + fn strict_decoder_rejects_ambiguous_names_modes_and_truncation() { + for bad in [ + entry(b"100644", b".."), + entry(b"100644", b"a/b"), + entry(b"100644", &[0xff]), + entry(b"100600", b"x"), + [entry(b"100644", b"x"), entry(b"100755", b"x")].concat(), + b"100644 x\0short".to_vec(), + b"100644".to_vec(), + ] { + assert!(decode_tree(&bad).is_err()); + } + assert!(decode_tree(&[]).unwrap().is_empty()); + assert_eq!( + decode_tree(&entry(b"120000", b"link")).unwrap()[0].kind, + EntryKind::Symlink + ); + assert_eq!( + decode_tree(&entry(b"160000", b"sub")).unwrap()[0].kind, + EntryKind::Gitlink + ); + } + + #[test] + fn verification_uses_git_type_and_length_header() { + let _guard = + git_internal::hash::set_hash_kind_for_test(git_internal::hash::HashKind::Sha256); + let empty = ObjectId::new("4b825dc642cb6eb9a060e54bf8d69288fbee4904").unwrap(); + verify_object(ObjectKind::Tree, &empty, &[]).unwrap(); + assert!(verify_object(ObjectKind::Blob, &empty, &[]).is_err()); + assert!(verify_object(ObjectKind::Tree, &empty, b"changed").is_err()); + } +} diff --git a/ceres/src/application/snapshot/radix.rs b/ceres/src/application/snapshot/radix.rs new file mode 100644 index 000000000..7d6ed47a1 --- /dev/null +++ b/ceres/src/application/snapshot/radix.rs @@ -0,0 +1,437 @@ +//! Persistent, compressed byte-radix index from canonical repository paths to +//! immutable binding digests. No live registry reads, mutable nodes or recursion. +//! Public pagination/authentication belongs above this internal index. + +use std::{collections::BTreeMap, sync::Arc}; + +use async_trait::async_trait; +use common::errors::MegaError; +use sha2::{Digest, Sha256}; + +use crate::model::snapshot::{MAX_PATH_BYTES, ManifestDigest, RepoPath}; + +pub const NODE_DOMAIN: &[u8] = b"mega.namespace-radix.v1\0"; +pub mod database; +pub const MAX_NODE_BYTES: usize = 16 * 1024; +pub const MAX_PAGE_SIZE: usize = 256; + +#[async_trait] +pub trait NodeStore: Send + Sync { + async fn read(&self, digest: &ManifestDigest) -> Result, MegaError>; + /// Insert-only, durable before the enclosing publication transaction commits. + /// A store must reject the same digest with different bytes, never overwrite. + async fn write(&self, digest: &ManifestDigest, bytes: &[u8]) -> Result<(), MegaError>; +} + +pub fn digest(bytes: &[u8]) -> ManifestDigest { + ManifestDigest::new(format!("sha256:{}", hex::encode(Sha256::digest(bytes)))) + .expect("SHA-256 yields a canonical digest") +} + +/// The empty index is a well-known, implicit node; no DB row is required. +pub fn empty_root() -> ManifestDigest { + digest(&Node::empty().encode()) +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct Node { + label: Vec, + value: Option, + children: BTreeMap, +} + +impl Node { + fn empty() -> Self { + Self { + label: Vec::new(), + value: None, + children: BTreeMap::new(), + } + } + fn leaf(label: &[u8], value: ManifestDigest) -> Self { + Self { + label: label.to_vec(), + value: Some(value), + children: BTreeMap::new(), + } + } + + fn encode(&self) -> Vec { + let mut out = NODE_DOMAIN.to_vec(); + out.extend_from_slice(&(self.label.len() as u16).to_be_bytes()); + out.extend_from_slice(&self.label); + out.push(u8::from(self.value.is_some())); + if let Some(value) = &self.value { + out.extend_from_slice(&raw_digest(value)); + } + out.extend_from_slice(&(self.children.len() as u16).to_be_bytes()); + for (edge, child) in &self.children { + out.push(*edge); + out.extend_from_slice(&raw_digest(child)); + } + out + } + + fn decode(bytes: &[u8]) -> Result { + if bytes.len() > MAX_NODE_BYTES { + return Err(corrupt("node byte limit")); + } + let mut input = bytes + .strip_prefix(NODE_DOMAIN) + .ok_or_else(|| corrupt("node schema"))?; + let label_len = read_u16(&mut input)?; + if label_len > MAX_PATH_BYTES { + return Err(corrupt("label byte limit")); + } + let label = take(&mut input, label_len)?.to_vec(); + let value = match take(&mut input, 1)?[0] { + 0 => None, + 1 => Some(read_digest(&mut input)?), + _ => return Err(corrupt("value tag")), + }; + let count = read_u16(&mut input)?; + if count > 256 { + return Err(corrupt("fanout limit")); + } + let mut children = BTreeMap::new(); + let mut previous = None; + for _ in 0..count { + let edge = take(&mut input, 1)?[0]; + if previous.is_some_and(|old| edge <= old) { + return Err(corrupt("unsorted or duplicate edge")); + } + children.insert(edge, read_digest(&mut input)?); + previous = Some(edge); + } + if !input.is_empty() + || (value.is_none() && count == 1) + || (value.is_none() && count == 0 && !label.is_empty()) + { + return Err(corrupt("noncanonical node")); + } + Ok(Self { + label, + value, + children, + }) + } +} + +fn raw_digest(value: &ManifestDigest) -> Vec { + hex::decode(&value.as_str()[7..]).expect("validated digest") +} +fn read_digest(input: &mut &[u8]) -> Result { + Ok(ManifestDigest::new(format!("sha256:{}", hex::encode(take(input, 32)?))).expect("32 bytes")) +} +fn read_u16(input: &mut &[u8]) -> Result { + let bytes = take(input, 2)?; + Ok(u16::from_be_bytes([bytes[0], bytes[1]]) as usize) +} +fn take<'a>(input: &mut &'a [u8], count: usize) -> Result<&'a [u8], MegaError> { + if input.len() < count { + return Err(corrupt("truncated node")); + } + let (head, tail) = input.split_at(count); + *input = tail; + Ok(head) +} +fn corrupt(detail: &str) -> MegaError { + MegaError::Unavailable(format!("namespace index integrity failure: {detail}")) +} + +/// A trailing NUL marks each component. Parent paths are key prefixes but +/// prefix neighbors (/rust and /rust_v1) are never ancestor bindings. +fn key(path: &RepoPath) -> Vec { + if path.as_str() == "/" { + return Vec::new(); + } + let mut bytes = path.as_str().as_bytes()[1..].to_vec(); + for byte in &mut bytes { + if *byte == b'/' { + *byte = 0; + } + } + bytes.push(0); + bytes +} +fn path_from_key(key: &[u8]) -> Result { + if key.is_empty() { + return Ok(RepoPath::new("/").expect("root")); + } + if key.last() != Some(&0) { + return Err(corrupt("value outside component boundary")); + } + let mut bytes = vec![b'/']; + bytes.extend( + key[..key.len() - 1] + .iter() + .map(|b| if *b == 0 { b'/' } else { *b }), + ); + let text = String::from_utf8(bytes).map_err(|_| corrupt("non-UTF-8 key"))?; + RepoPath::new(text).map_err(|_| corrupt("invalid key path")) +} + +pub struct RadixIndex<'a> { + store: &'a dyn NodeStore, +} + +#[derive(Debug, PartialEq, Eq)] +pub struct IndexPage { + pub entries: Vec<(RepoPath, ManifestDigest)>, + pub has_more: bool, +} + +impl<'a> RadixIndex<'a> { + pub fn new(store: &'a dyn NodeStore) -> Self { + Self { store } + } + + async fn load(&self, id: &ManifestDigest, edge: Option) -> Result { + let node = if id == &empty_root() { + Node::empty() + } else { + let bytes = self.store.read(id).await?; + if bytes.len() > MAX_NODE_BYTES { + return Err(corrupt("node byte limit")); + } + if digest(&bytes) != *id { + return Err(corrupt("node digest")); + } + Node::decode(&bytes)? + }; + if edge.is_some_and(|edge| node.label.first() != Some(&edge)) { + return Err(corrupt("child label does not match edge")); + } + Ok(node) + } + + async fn save(&self, mut node: Node) -> Result, MegaError> { + if node.value.is_none() { + if node.children.is_empty() { + return Ok(None); + } + if node.children.len() == 1 { + let (edge, id) = node.children.first_key_value().expect("one child"); + let child = self.load(id, Some(*edge)).await?; + node.label.extend_from_slice(&child.label); + node.value = child.value; + node.children = child.children; + } + } + if node.label.len() > MAX_PATH_BYTES { + return Err(corrupt("combined label limit")); + } + let bytes = node.encode(); + if bytes.len() > MAX_NODE_BYTES { + return Err(corrupt("encoded node limit")); + } + let id = digest(&bytes); + self.store.write(&id, &bytes).await?; + Ok(Some(id)) + } + + /// Copy only changed ancestors; a no-op returns the same root without writes. + /// The old root remains valid. Deletion canonicalizes compressed edges, so + /// insert order and a delete/reinsert roundtrip cannot change the digest. + pub async fn update( + &self, + root: &ManifestDigest, + path: &RepoPath, + value: Option, + ) -> Result { + let key = key(path); + let mut offset = 0; + let mut id = root.clone(); + let mut edge = None; + let mut ancestors = Vec::new(); + let mut replacement; + loop { + let mut node = self.load(&id, edge).await?; + let remaining = &key[offset..]; + let common = remaining + .iter() + .zip(&node.label) + .take_while(|(a, b)| a == b) + .count(); + if common < node.label.len() { + let Some(value) = value else { + return Ok(root.clone()); + }; + let mut parent = Node { + label: node.label[..common].to_vec(), + ..Node::empty() + }; + node.label.drain(..common); + parent.children.insert( + node.label[0], + self.save(node).await?.expect("existing nonempty node"), + ); + if common == remaining.len() { + parent.value = Some(value); + } else { + let suffix = &remaining[common..]; + parent.children.insert( + suffix[0], + self.save(Node::leaf(suffix, value)).await?.expect("leaf"), + ); + } + replacement = self.save(parent).await?; + break; + } + offset += common; + if offset == key.len() { + if node.value == value { + return Ok(root.clone()); + } + node.value = value; + replacement = self.save(node).await?; + break; + } + let next_edge = key[offset]; + if let Some(child) = node.children.get(&next_edge) { + id = child.clone(); + edge = Some(next_edge); + ancestors.push((node, next_edge)); + } else { + let Some(value) = value else { + return Ok(root.clone()); + }; + node.children.insert( + next_edge, + self.save(Node::leaf(&key[offset..], value)) + .await? + .expect("leaf"), + ); + replacement = self.save(node).await?; + break; + } + } + while let Some((mut parent, edge)) = ancestors.pop() { + match replacement { + Some(id) => { + parent.children.insert(edge, id); + } + None => { + parent.children.remove(&edge); + } + } + replacement = self.save(parent).await?; + } + Ok(replacement.unwrap_or_else(empty_root)) + } + + pub async fn get( + &self, + root: &ManifestDigest, + path: &RepoPath, + ) -> Result, MegaError> { + Ok(self.walk(root, path, false).await?.map(|(_, value)| value)) + } + + pub async fn longest_prefix( + &self, + root: &ManifestDigest, + path: &RepoPath, + ) -> Result, MegaError> { + self.walk(root, path, true).await + } + + async fn walk( + &self, + root: &ManifestDigest, + path: &RepoPath, + ancestors: bool, + ) -> Result, MegaError> { + let key = key(path); + let mut id = root.clone(); + let mut edge = None; + let mut offset = 0; + let mut found = None; + loop { + let node = self.load(&id, edge).await?; + if !key[offset..].starts_with(&node.label) { + break; + } + offset += node.label.len(); + if let Some(value) = node.value { + found = Some((path_from_key(&key[..offset])?, value)); + } + if offset == key.len() { + return Ok(found.filter(|(p, _)| ancestors || p == path)); + } + let next_edge = key[offset]; + let Some(child) = node.children.get(&next_edge) else { + break; + }; + id = child.clone(); + edge = Some(next_edge); + } + Ok(if ancestors { found } else { None }) + } + + /// Internal keyset page in encoded-component order. HTTP must authenticate + /// its cursor and bind it to view/prefix/query/schema; raw `after` is not an + /// externally trustworthy cursor. Only intersecting subtrees are visited. + pub async fn page( + &self, + root: &ManifestDigest, + prefix: &RepoPath, + after: Option<&RepoPath>, + limit: usize, + ) -> Result { + if limit == 0 + || limit > MAX_PAGE_SIZE + || after.is_some_and(|p| p.relative_to(prefix).is_none()) + { + return Err(MegaError::bad_request("invalid namespace index page")); + } + let prefix = key(prefix); + let after = after.map(key); + let mut stack = vec![(root.clone(), Arc::<[u8]>::from([]), None)]; + let mut entries = Vec::new(); + while let Some((id, parent, edge)) = stack.pop() { + let node = self.load(&id, edge).await?; + let full: Arc<[u8]> = [parent.as_ref(), &node.label].concat().into(); + if full.len() > MAX_PATH_BYTES { + return Err(corrupt("key byte limit")); + } + if !intersects(&full, &prefix, after.as_deref()) { + continue; + } + if let Some(value) = node.value + && full.starts_with(&prefix) + && after + .as_ref() + .is_none_or(|after| full.as_ref() > after.as_slice()) + { + entries.push((path_from_key(&full)?, value)); + if entries.len() > limit { + entries.pop(); + return Ok(IndexPage { + entries, + has_more: true, + }); + } + } + for (edge, child) in node.children.into_iter().rev() { + let mut lower_bound = full.to_vec(); + lower_bound.push(edge); + if intersects(&lower_bound, &prefix, after.as_deref()) { + stack.push((child, full.clone(), Some(edge))); + } + } + } + Ok(IndexPage { + entries, + has_more: false, + }) + } +} + +fn intersects(candidate: &[u8], prefix: &[u8], after: Option<&[u8]>) -> bool { + (candidate.starts_with(prefix) || prefix.starts_with(candidate)) + && !after.is_some_and(|after| candidate < after && !after.starts_with(candidate)) +} + +#[cfg(test)] +mod tests; diff --git a/ceres/src/application/snapshot/radix/database.rs b/ceres/src/application/snapshot/radix/database.rs new file mode 100644 index 000000000..c934ed74f --- /dev/null +++ b/ceres/src/application/snapshot/radix/database.rs @@ -0,0 +1,93 @@ +//! NodeStore adapter that borrows the caller's connection or transaction. + +use jupiter::{sea_orm::ConnectionTrait, storage::namespace_storage::NamespaceStorage}; + +use super::*; + +pub struct DatabaseNodeStore<'a, C: ConnectionTrait> { + storage: &'a NamespaceStorage, + connection: &'a C, +} + +impl<'a, C: ConnectionTrait> DatabaseNodeStore<'a, C> { + pub fn new(storage: &'a NamespaceStorage, connection: &'a C) -> Self { + Self { + storage, + connection, + } + } +} + +#[async_trait] +impl NodeStore for DatabaseNodeStore<'_, C> { + async fn read(&self, id: &ManifestDigest) -> Result, MegaError> { + self.storage + .node_in(self.connection, id.as_str()) + .await? + .ok_or_else(|| MegaError::Unavailable("namespace index node unavailable".into())) + } + + async fn write(&self, id: &ManifestDigest, bytes: &[u8]) -> Result<(), MegaError> { + self.storage + .put_node_in(self.connection, id.as_str(), bytes) + .await + } +} + +#[cfg(test)] +mod postgres_tests; + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use jupiter::storage::base_storage::{BaseStorage, StorageConnector}; + use sea_orm::TransactionTrait; + use tempfile::TempDir; + + use super::*; + + #[tokio::test] + async fn radix_database_nodes_follow_the_outer_transaction_and_reopen() { + let dir = TempDir::new().unwrap(); + let conn = jupiter::tests::test_db_connection(dir.path()).await; + jupiter_migrate::apply_migrations(&conn, true) + .await + .unwrap(); + let storage = NamespaceStorage { + base: BaseStorage::new(Arc::new(conn)), + }; + let path = RepoPath::new("/third-party/r").unwrap(); + let old = digest(b"old binding"); + let conn = storage.base.get_connection(); + let txn = conn.begin().await.unwrap(); + let db = DatabaseNodeStore::new(&storage, &txn); + let index = RadixIndex::new(&db); + let root = index + .update(&empty_root(), &path, Some(old.clone())) + .await + .unwrap(); + assert_eq!(index.get(&root, &path).await.unwrap(), Some(old.clone())); + txn.rollback().await.unwrap(); + assert!(storage.node(root.as_str()).await.unwrap().is_none()); + let txn = conn.begin().await.unwrap(); + let db = DatabaseNodeStore::new(&storage, &txn); + let committed = RadixIndex::new(&db) + .update(&empty_root(), &path, Some(old.clone())) + .await + .unwrap(); + assert_eq!(committed, root); + txn.commit().await.unwrap(); + let reopened = sea_orm::Database::connect(format!( + "sqlite://{}", + dir.path().join("test.db").display() + )) + .await + .unwrap(); + let db = DatabaseNodeStore::new(&storage, &reopened); + assert_eq!( + RadixIndex::new(&db).get(&root, &path).await.unwrap(), + Some(old) + ); + } +} diff --git a/ceres/src/application/snapshot/radix/database/postgres_tests.rs b/ceres/src/application/snapshot/radix/database/postgres_tests.rs new file mode 100644 index 000000000..52d521962 --- /dev/null +++ b/ceres/src/application/snapshot/radix/database/postgres_tests.rs @@ -0,0 +1,140 @@ +//! Explicit opt-in PostgreSQL gate. Always creates a fresh, randomly named test +//! schema, never refreshes or drops a supplied database. Use a disposable server. + +use std::sync::Arc; + +use jupiter::storage::{ + base_storage::{BaseStorage, StorageConnector}, + snapshot_storage::{ScopeAttestation, ScopeProofKind, SnapshotStorage, SourceKind}, +}; +use sea_orm::{ConnectOptions, ConnectionTrait, Database, TransactionTrait}; + +use super::*; + +#[tokio::test] +#[ignore = "requires explicit MEGA_SNAPSHOT_TEST_DATABASE_URL pointing to a disposable loopback PostgreSQL test database"] +async fn postgres_snapshot_nodes_scope_proofs_and_radix_transactions() { + let url = std::env::var("MEGA_SNAPSHOT_TEST_DATABASE_URL") + .expect("set explicit disposable PostgreSQL test URL"); + let parsed = reqwest::Url::parse(&url).unwrap(); + assert!(matches!(parsed.scheme(), "postgres" | "postgresql")); + assert!(matches!( + parsed.host_str(), + Some("localhost" | "127.0.0.1" | "[::1]") + )); + assert_eq!( + parsed.path(), + "/snapshot_test", + "use the explicit disposable test database" + ); + let schema = format!("snapshot_test_{}", uuid::Uuid::new_v4().simple()); + let control = Database::connect( + ConnectOptions::new(url.clone()) + .max_connections(1) + .sqlx_logging(false) + .to_owned(), + ) + .await + .unwrap(); + // The interpolated identifier is entirely generated above (ASCII hex). + control + .execute_unprepared(&format!("CREATE SCHEMA {schema}")) + .await + .unwrap(); + let options = ConnectOptions::new(url) + .max_connections(16) + .sqlx_logging(false) + .set_schema_search_path(schema.clone()) + .to_owned(); + let connection = Database::connect(options.clone()).await.unwrap(); + jupiter_migrate::apply_migrations(&connection, false) + .await + .unwrap(); + let base = BaseStorage::new(Arc::new(connection)); + let nodes = NamespaceStorage { base: base.clone() }; + let sources = SnapshotStorage { base }; + let conn = nodes.base.get_connection(); + + let registered = + futures::future::join_all((0..16).map(|_| sources.ensure_source(SourceKind::Import, 42))) + .await + .into_iter() + .map(Result::unwrap) + .collect::>(); + assert!( + registered + .iter() + .all(|s| s.source_id == registered[0].source_id) + ); + let proof = ScopeAttestation { + source_id: registered[0].source_id.clone(), + scope_path: format!("/{}", vec!["a".repeat(250); 15].join("/")), + commit_oid: "1".repeat(40), + root_tree_oid: "2".repeat(40), + proof_kind: ScopeProofKind::ImportCommit, + proof_oid: None, + }; + let txn = conn.begin().await.unwrap(); + sources.record_scope_in(&txn, &proof).await.unwrap(); + let adapter = DatabaseNodeStore::new(&nodes, &txn); + let index = RadixIndex::new(&adapter); + let path = RepoPath::new("/third-party/r").unwrap(); + let old = digest(b"binding A"); + let root = index + .update(&empty_root(), &path, Some(old.clone())) + .await + .unwrap(); + txn.rollback().await.unwrap(); + assert!(nodes.node(root.as_str()).await.unwrap().is_none()); + assert!( + sources + .scope(&proof.source_id, &proof.scope_path, &proof.commit_oid) + .await + .unwrap() + .is_none() + ); + + let txn = conn.begin().await.unwrap(); + sources.record_scope_in(&txn, &proof).await.unwrap(); + let adapter = DatabaseNodeStore::new(&nodes, &txn); + let committed = RadixIndex::new(&adapter) + .update(&empty_root(), &path, Some(old.clone())) + .await + .unwrap(); + assert_eq!(committed, root); + txn.commit().await.unwrap(); + let bad_proof = ScopeAttestation { + root_tree_oid: "3".repeat(40), + ..proof.clone() + }; + assert!(sources.record_scope_in(conn, &bad_proof).await.is_err()); + let reopened = Database::connect(options).await.unwrap(); + let adapter = DatabaseNodeStore::new(&nodes, &reopened); + let index = RadixIndex::new(&adapter); + assert_eq!(index.get(&root, &path).await.unwrap(), Some(old.clone())); + let new_root = index + .update(&root, &path, Some(digest(b"binding B"))) + .await + .unwrap(); + assert_eq!(index.get(&root, &path).await.unwrap(), Some(old)); + assert_ne!(root, new_root); + let bytes = vec![42; MAX_NODE_BYTES]; + let id = digest(&bytes); + let writes = + futures::future::join_all((0..16).map(|_| nodes.put_node_in(conn, id.as_str(), &bytes))) + .await; + for write in writes { + write.unwrap(); + } + assert_eq!(nodes.node(id.as_str()).await.unwrap().unwrap(), bytes); + let oversized = vec![42; MAX_NODE_BYTES + 1]; + assert!( + nodes + .put_node_in(conn, digest(&oversized).as_str(), &oversized) + .await + .is_err() + ); + println!( + "PostgreSQL snapshot/node migrations, concurrent source/node inserts, 3765-byte scope key, rollback, reopened reads and immutable root A/B passed; test schema retained: {schema}" + ); +} diff --git a/ceres/src/application/snapshot/radix/tests.rs b/ceres/src/application/snapshot/radix/tests.rs new file mode 100644 index 000000000..02cf43688 --- /dev/null +++ b/ceres/src/application/snapshot/radix/tests.rs @@ -0,0 +1,464 @@ +use std::{ + collections::HashMap, + sync::{ + Mutex, + atomic::{AtomicUsize, Ordering}, + }, +}; + +use super::*; + +#[derive(Default)] +struct MemoryStore { + nodes: Mutex>>, + reads: AtomicUsize, + writes: AtomicUsize, + read_bytes: AtomicUsize, + write_bytes: AtomicUsize, + largest: AtomicUsize, +} + +#[async_trait] +impl NodeStore for MemoryStore { + async fn read(&self, id: &ManifestDigest) -> Result, MegaError> { + self.reads.fetch_add(1, Ordering::Relaxed); + let bytes = self + .nodes + .lock() + .unwrap() + .get(id) + .cloned() + .ok_or_else(|| corrupt("missing node"))?; + self.read_bytes.fetch_add(bytes.len(), Ordering::Relaxed); + Ok(bytes) + } + async fn write(&self, id: &ManifestDigest, bytes: &[u8]) -> Result<(), MegaError> { + assert_eq!(&digest(bytes), id); + assert!(bytes.len() <= MAX_NODE_BYTES); + let mut nodes = self.nodes.lock().unwrap(); + if let Some(old) = nodes.get(id) { + assert_eq!(old, bytes); + } + nodes.insert(id.clone(), bytes.to_vec()); + self.writes.fetch_add(1, Ordering::Relaxed); + self.write_bytes.fetch_add(bytes.len(), Ordering::Relaxed); + self.largest.fetch_max(bytes.len(), Ordering::Relaxed); + Ok(()) + } +} + +impl MemoryStore { + fn reset_metrics(&self) { + for metric in [ + &self.reads, + &self.writes, + &self.read_bytes, + &self.write_bytes, + &self.largest, + ] { + metric.store(0, Ordering::Relaxed); + } + } +} +fn p(path: &str) -> RepoPath { + RepoPath::new(path).unwrap() +} +fn v(value: &str) -> ManifestDigest { + digest(value.as_bytes()) +} + +#[test] +fn canonical_vectors_match_independent_dotnet_encoding() { + #[derive(serde::Deserialize)] + struct Vector { + name: String, + canonical_hex: String, + digest: ManifestDigest, + } + let vectors: Vec = serde_json::from_str(include_str!( + "../../../../tests/fixtures/snapshot/namespace-radix-v1.json" + )) + .unwrap(); + for vector in vectors { + let bytes = hex::decode(vector.canonical_hex).unwrap(); + assert_eq!(digest(&bytes), vector.digest, "{}", vector.name); + assert_eq!(Node::decode(&bytes).unwrap().encode(), bytes); + if vector.name == "empty" { + assert_eq!(empty_root(), vector.digest); + } + } +} + +#[tokio::test] +async fn index_is_canonical_independent_of_insert_order_and_preserves_old_roots() { + let store = MemoryStore::default(); + let index = RadixIndex::new(&store); + let paths = [ + "/", + "/third-party/rust", + "/third-party/rust_v1", + "/third-party/rust/crate", + "/project/库+1", + "/project/ab", + "/project/a", + ]; + let mut root = empty_root(); + for path in paths { + root = index.update(&root, &p(path), Some(v(path))).await.unwrap(); + } + let mut reverse = empty_root(); + for path in paths.into_iter().rev() { + reverse = index + .update(&reverse, &p(path), Some(v(path))) + .await + .unwrap(); + } + assert_eq!(root, reverse); + let changed = index + .update(&root, &p(paths[1]), Some(v("new"))) + .await + .unwrap(); + assert_ne!(root, changed); + for path in paths { + assert_eq!(index.get(&root, &p(path)).await.unwrap(), Some(v(path))); + } + assert_eq!( + index.get(&changed, &p(paths[1])).await.unwrap(), + Some(v("new")) + ); + let restored = index + .update(&changed, &p(paths[1]), Some(v(paths[1]))) + .await + .unwrap(); + assert_eq!(restored, root); + for path in paths { + reverse = index.update(&reverse, &p(path), None).await.unwrap(); + } + assert_eq!(reverse, empty_root()); + store.reset_metrics(); + assert_eq!( + index.update(&root, &p("/absent"), None).await.unwrap(), + root + ); + assert_eq!( + index + .update(&root, &p(paths[1]), Some(v(paths[1]))) + .await + .unwrap(), + root + ); + assert_eq!(store.writes.load(Ordering::Relaxed), 0); +} + +#[tokio::test] +async fn pages_stay_on_the_requested_immutable_root_and_component_prefix() { + let store = MemoryStore::default(); + let index = RadixIndex::new(&store); + let mut root = empty_root(); + let paths = [ + "/rust", + "/rust/crate", + "/rust/a", + "/rust/a/b", + "/rust_v1", + "/rusty/a", + ]; + for path in paths { + root = index.update(&root, &p(path), Some(v(path))).await.unwrap(); + } + assert_eq!( + index + .longest_prefix(&root, &p("/rust/crate/src/lib.rs")) + .await + .unwrap(), + Some((p("/rust/crate"), v("/rust/crate"))) + ); + assert_eq!( + index + .longest_prefix(&root, &p("/rustz/crate")) + .await + .unwrap(), + None + ); + assert_eq!(index.get(&root, &p("/rust/crate/src")).await.unwrap(), None); + let newer = index + .update(&root, &p("/rust/new"), Some(v("new"))) + .await + .unwrap(); + let first = index.page(&root, &p("/rust"), None, 2).await.unwrap(); + assert!(first.has_more); + let after = &first.entries.last().unwrap().0; + let second = index + .page(&root, &p("/rust"), Some(after), 2) + .await + .unwrap(); + assert!(!second.has_more); + assert_eq!( + first + .entries + .into_iter() + .chain(second.entries) + .map(|(p, _)| p) + .collect::>(), + vec![p("/rust"), p("/rust/a"), p("/rust/a/b"), p("/rust/crate")] + ); + assert_eq!( + index + .page(&newer, &p("/rust"), None, 10) + .await + .unwrap() + .entries + .len(), + 5 + ); + assert!( + index + .page(&root, &p("/rust"), Some(&p("/rust_v1")), 10) + .await + .is_err() + ); + assert!(index.page(&root, &p("/"), None, 0).await.is_err()); + assert!( + index + .page(&root, &p("/"), None, MAX_PAGE_SIZE + 1) + .await + .is_err() + ); +} + +#[tokio::test] +async fn deterministic_mutation_trace_matches_an_independent_ordered_map() { + let store = MemoryStore::default(); + let index = RadixIndex::new(&store); + let mut root = empty_root(); + let mut oracle = BTreeMap::new(); + let mut seed = 0x1234u64; + for step in 0..800 { + seed = seed.wrapping_mul(6364136223846793005).wrapping_add(1); + let path = p(&format!("/pkg/{:03}/crate", (seed >> 32) % 90)); + let value = (!seed.is_multiple_of(5)).then(|| v(&format!("{step}"))); + root = index.update(&root, &path, value.clone()).await.unwrap(); + match value { + Some(value) => { + oracle.insert(path.clone(), value); + } + None => { + oracle.remove(&path); + } + } + assert_eq!( + index.get(&root, &path).await.unwrap(), + oracle.get(&path).cloned() + ); + if step % 25 == 0 { + let page = index.page(&root, &p("/pkg"), None, 100).await.unwrap(); + assert!(!page.has_more); + assert_eq!( + page.entries, + oracle + .iter() + .map(|(p, v)| (p.clone(), v.clone())) + .collect::>() + ); + } + } +} + +#[tokio::test] +async fn corruption_missing_nodes_and_wrong_edge_labels_are_not_empty_successes() { + let store = MemoryStore::default(); + let index = RadixIndex::new(&store); + let root = index + .update(&empty_root(), &p("/a"), Some(v("A"))) + .await + .unwrap(); + store + .nodes + .lock() + .unwrap() + .insert(root.clone(), b"bad data".to_vec()); + assert!(index.get(&root, &p("/a")).await.is_err()); + assert!(index.get(&v("absent"), &p("/a")).await.is_err()); + let child = index + .save(Node::leaf(b"b\0", v("B"))) + .await + .unwrap() + .unwrap(); + let bytes = Node { + label: Vec::new(), + value: Some(v("root")), + children: BTreeMap::from([(b'a', child)]), + } + .encode(); + let wrong = digest(&bytes); + store.write(&wrong, &bytes).await.unwrap(); + assert!(index.get(&wrong, &p("/a")).await.is_err()); +} + +#[test] +fn node_codec_enforces_size_fanout_sorting_and_canonical_compression() { + let node = Node { + label: vec![b'a'; MAX_PATH_BYTES], + value: Some(v("value")), + children: (0..=255) + .map(|edge| (edge, v(&format!("{edge}")))) + .collect(), + }; + let bytes = node.encode(); + assert!(bytes.len() <= MAX_NODE_BYTES); + assert_eq!(Node::decode(&bytes).unwrap(), node); + assert!(Node::decode(&vec![0; MAX_NODE_BYTES + 1]).is_err()); + assert!(Node::decode(&bytes[..bytes.len() - 1]).is_err()); + let single = Node { + label: b"a".to_vec(), + value: None, + children: BTreeMap::from([(b'b', v("b"))]), + }; + assert!(Node::decode(&single.encode()).is_err()); + let mut trailing = Node::empty().encode(); + trailing.push(0); + assert!(Node::decode(&trailing).is_err()); + let mut bad_count = Node::empty().encode(); + let len = bad_count.len(); + bad_count[len - 2..].copy_from_slice(&257u16.to_be_bytes()); + assert!(Node::decode(&bad_count).is_err()); +} + +#[tokio::test] +async fn maximum_path_and_long_shared_prefix_do_not_recurse_or_rewrite_siblings() { + let store = MemoryStore::default(); + let index = RadixIndex::new(&store); + let prefix = format!("/{}/", vec!["a".repeat(255); 15].join("/")); + let path = p(&format!("{prefix}{}", "b".repeat(255))); + assert_eq!(path.as_str().len(), MAX_PATH_BYTES); + let first = index + .update(&empty_root(), &path, Some(v("old"))) + .await + .unwrap(); + let neighbor = p(&format!("{prefix}{}c", "b".repeat(254))); + let root = index + .update(&first, &neighbor, Some(v("neighbor"))) + .await + .unwrap(); + store.reset_metrics(); + let changed = index.update(&root, &path, Some(v("new"))).await.unwrap(); + assert!(store.writes.load(Ordering::Relaxed) <= 3); + assert!(store.largest.load(Ordering::Relaxed) <= MAX_NODE_BYTES); + assert_eq!(index.get(&root, &path).await.unwrap(), Some(v("old"))); + assert_eq!( + index.get(&changed, &neighbor).await.unwrap(), + Some(v("neighbor")) + ); +} + +// Independent structured fixture generation avoids retaining a million +// intermediate publication roots during initial construction. It does not call +// update/get/page to determine expected mappings; decimal keys are the oracle. +fn decimal_fixture(store: &MemoryStore, digits: usize) -> ManifestDigest { + fn build( + nodes: &mut HashMap>, + depth: usize, + digits: usize, + number: usize, + label: Vec, + ) -> ManifestDigest { + let node = if depth == digits { + let mut label = label; + label.push(0); + Node::leaf(&label, v(&format!("binding-{number}"))) + } else { + Node { + label, + value: None, + children: (0..10) + .map(|n| { + let edge = b'0' + n as u8; + ( + edge, + build(nodes, depth + 1, digits, number * 10 + n, vec![edge]), + ) + }) + .collect(), + } + }; + let bytes = node.encode(); + let id = digest(&bytes); + nodes.insert(id.clone(), bytes); + id + } + build( + &mut store.nodes.lock().unwrap(), + 0, + digits, + 0, + b"third-party\0r".to_vec(), + ) +} + +async fn scale_test(digits: usize) { + let store = MemoryStore::default(); + let root = decimal_fixture(&store, digits); + let index = RadixIndex::new(&store); + let number = 10usize.pow(digits as u32) / 2 + 17; + let path = p(&format!("/third-party/r{number:0digits$}")); + store.reset_metrics(); + let updated = index + .update(&root, &path, Some(v("updated"))) + .await + .unwrap(); + let update_reads = store.reads.load(Ordering::Relaxed); + let update_writes = store.writes.load(Ordering::Relaxed); + let read_bytes = store.read_bytes.load(Ordering::Relaxed); + let write_bytes = store.write_bytes.load(Ordering::Relaxed); + let largest_node = store.largest.load(Ordering::Relaxed); + assert!(update_reads <= digits + 2 && update_writes <= digits + 2); + assert_eq!( + index.get(&root, &path).await.unwrap(), + Some(v(&format!("binding-{number}"))) + ); + assert_eq!( + index.get(&updated, &path).await.unwrap(), + Some(v("updated")) + ); + store.reset_metrics(); + let page = index + .page(&root, &p("/third-party"), Some(&path), 32) + .await + .unwrap(); + assert!(page.has_more); + let page_reads = store.reads.load(Ordering::Relaxed); + assert!(page_reads < 100); + for (offset, (path, value)) in page.entries.iter().enumerate() { + let n = number + offset + 1; + assert_eq!(path, &p(&format!("/third-party/r{n:0digits$}"))); + assert_eq!(value, &v(&format!("binding-{n}"))); + } + store.reset_metrics(); + let one = index.page(&root, &path, None, 1).await.unwrap(); + assert_eq!(one.entries.len(), 1); + assert!(!one.has_more); + let prefix_reads = store.reads.load(Ordering::Relaxed); + assert!(prefix_reads <= digits + 2); + let peak = std::fs::read_to_string("/proc/self/status") + .ok() + .and_then(|s| { + s.lines() + .find(|l| l.starts_with("VmHWM:")) + .map(str::to_owned) + }); + println!( + "bindings={} update_reads={update_reads} update_writes={update_writes} read_bytes={read_bytes} write_bytes={write_bytes} largest_node={largest_node} page32_reads={page_reads} single_prefix_reads={prefix_reads} process_peak={peak:?}", + 10usize.pow(digits as u32) + ); +} + +#[tokio::test] +async fn ten_thousand_bindings_keep_updates_and_pages_bounded() { + scale_test(4).await; +} + +#[tokio::test] +#[ignore = "explicit scale gate: cargo test -p ceres --lib snapshot::radix::tests::million -- --ignored --nocapture"] +async fn million_bindings_keep_updates_and_pages_bounded() { + scale_test(6).await; +} diff --git a/ceres/src/application/snapshot/source.rs b/ceres/src/application/snapshot/source.rs new file mode 100644 index 000000000..c6eb74914 --- /dev/null +++ b/ceres/src/application/snapshot/source.rs @@ -0,0 +1,135 @@ +use std::{collections::HashSet, str::FromStr}; + +use common::errors::MegaError; +use git_internal::{ + hash::{HashKind, ObjectHash}, + internal::object::{ObjectTrait, tree::Tree}, +}; +use jupiter::storage::git_db_storage::GitDbStorage; +use sha1::{Digest, Sha1}; + +/// A resolved import root never consults a moving ref again. +/// This internal value does not claim a namespace publication or a retention lease. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct ResolvedImportCommit { + pub repo_id: i64, + pub commit_oid: String, + pub root_tree_oid: String, +} + +/// Compatibility selector for existing code browsing APIs: absent/empty means +/// default, a full SHA-1 means commit, a fully qualified ref is exact, and an +/// unqualified name means a legacy tag (never an ambiguous branch fallback). +pub(crate) async fn resolve_import_commit( + storage: &GitDbStorage, + repo_id: i64, + reference: Option<&str>, +) -> Result { + let reference = reference.unwrap_or_default().trim(); + let (mut oid, peel_tags) = if reference.is_empty() { + let default_ref = storage + .get_unique_default_ref(repo_id) + .await? + .ok_or_else(|| MegaError::NotFound("default import ref not found".into()))?; + (default_ref.ref_git_id, false) + } else if reference.len() == 40 && reference.bytes().all(|b| b.is_ascii_hexdigit()) { + (reference.to_ascii_lowercase(), false) + } else { + let full_ref = + if reference.starts_with("refs/heads/") || reference.starts_with("refs/tags/") { + reference.to_owned() + } else if reference.starts_with("refs/") { + return Err(MegaError::bad_request("unsupported import ref namespace")); + } else { + format!("refs/tags/{reference}") + }; + let selected = storage + .get_ref_by_name(repo_id, &full_ref) + .await? + .ok_or_else(|| MegaError::NotFound(format!("import ref not found: {full_ref}")))?; + (selected.ref_git_id, full_ref.starts_with("refs/tags/")) + }; + + let mut seen = HashSet::new(); + // The final commit after 32 annotated tags is accepted; a 33rd tag is not. + for depth in 0..=32 { + if oid.len() != 40 || !oid.bytes().all(|b| b.is_ascii_hexdigit()) { + return Err(MegaError::Unavailable( + "invalid stored import object id".into(), + )); + } + oid.make_ascii_lowercase(); + if !seen.insert(oid.clone()) { + return Err(MegaError::bad_request("cyclic annotated tag chain")); + } + if let Some(commit) = storage.get_commit_by_hash(repo_id, &oid).await? { + let tree_oid = ObjectHash::from_str(&commit.tree) + .map_err(|_| MegaError::Unavailable("invalid stored import root tree id".into()))?; + if tree_oid.kind() != HashKind::Sha1 { + return Err(MegaError::Unavailable( + "import commit and tree hash formats differ".into(), + )); + } + return Ok(ResolvedImportCommit { + repo_id, + commit_oid: commit.commit_id, + root_tree_oid: commit.tree, + }); + } + if !peel_tags { + return Err(MegaError::NotFound(format!( + "import commit not found: {oid}" + ))); + } + let tag = storage + .get_tag_by_hash(repo_id, &oid) + .await? + .ok_or_else(|| { + MegaError::NotFound(format!("import tag target is not a commit or tag: {oid}")) + })?; + if tag.object_type != "commit" && tag.object_type != "tag" { + return Err(MegaError::bad_request("tag does not resolve to a commit")); + } + if depth == 32 { + return Err(MegaError::bad_request( + "annotated tag chain exceeds 32 objects", + )); + } + oid = tag.object_id; + } + unreachable!("the bounded tag loop returns at its depth limit") +} + +pub(crate) async fn read_import_root( + storage: &GitDbStorage, + source: &ResolvedImportCommit, +) -> Result { + let model = storage + .get_tree_by_hash(source.repo_id, &source.root_tree_oid) + .await? + .ok_or_else(|| { + MegaError::NotFound(format!("import tree not found: {}", source.root_tree_oid)) + })?; + let oid = ObjectHash::from_str(&model.tree_id) + .map_err(|_| MegaError::Unavailable("invalid stored import tree id".into()))?; + if oid.kind() != HashKind::Sha1 { + return Err(MegaError::Unavailable( + "unsupported import tree hash format".into(), + )); + } + // ObjectHash::new selects a thread-local algorithm, not the algorithm of + // this async request. Hash this SHA-1 source explicitly, including Git's header. + let mut hash = Sha1::new(); + hash.update(format!("tree {}\0", model.sub_trees.len()).as_bytes()); + hash.update(&model.sub_trees); + if hex::encode(hash.finalize()) != source.root_tree_oid { + return Err(MegaError::Unavailable( + "stored import tree hash mismatch".into(), + )); + } + Tree::from_bytes(&model.sub_trees, oid) + .map_err(|e| MegaError::Unavailable(format!("invalid stored import tree: {e}"))) +} + +#[cfg(test)] +mod tests; diff --git a/ceres/src/application/snapshot/source/tests.rs b/ceres/src/application/snapshot/source/tests.rs new file mode 100644 index 000000000..ec54f36f0 --- /dev/null +++ b/ceres/src/application/snapshot/source/tests.rs @@ -0,0 +1,323 @@ +use std::sync::Arc; + +use callisto::{ + git_commit, git_repo, git_tag, git_tree, import_refs, sea_orm_active_enums::RefTypeEnum, +}; +use git_internal::internal::{ + metadata::EntryMeta, + object::{ + blob::Blob, + commit::Commit, + tree::{Tree, TreeItem, TreeItemMode}, + }, +}; +use jupiter::{ + storage::base_storage::{BaseStorage, StorageConnector}, + utils::converter::IntoGitModel, +}; +use sea_orm::{ColumnTrait, EntityTrait, IntoActiveModel, QueryFilter, sea_query::Expr}; +use tempfile::TempDir; + +use super::*; + +#[tokio::test] +async fn corrupt_or_missing_root_is_an_error_not_an_empty_directory() { + let (_dir, storage) = fixture().await; + let (a, _) = commit(&storage, 101, "A").await; + let source = resolve_import_commit(&storage, 101, Some(&a.id.to_string())) + .await + .unwrap(); + git_tree::Entity::update_many() + .col_expr(git_tree::Column::SubTrees, Expr::value(Vec::::new())) + .filter(git_tree::Column::RepoId.eq(101)) + .exec(storage.get_connection()) + .await + .unwrap(); + assert!(matches!( + read_import_root(&storage, &source).await, + Err(MegaError::Unavailable(_)) + )); + git_tree::Entity::delete_many() + .filter(git_tree::Column::RepoId.eq(101)) + .exec(storage.get_connection()) + .await + .unwrap(); + assert!(matches!( + read_import_root(&storage, &source).await, + Err(MegaError::NotFound(_)) + )); +} + +#[tokio::test] +async fn annotated_tag_depth_is_bounded() { + let (_dir, storage) = fixture().await; + let (a, _) = commit(&storage, 101, "A").await; + let mut target = a.id.to_string(); + for n in 1..=33 { + let id = format!("{n:040x}"); + tag( + &storage, + &id, + &target, + if n == 1 { "commit" } else { "tag" }, + ) + .await; + target = id; + if n == 32 { + reference(&storage, 101, "refs/tags/allowed", &target, false).await; + } + } + reference(&storage, 101, "refs/tags/too-deep", &target, false).await; + assert_eq!( + resolve_import_commit(&storage, 101, Some("allowed")) + .await + .unwrap() + .commit_oid, + a.id.to_string() + ); + assert!(matches!( + resolve_import_commit(&storage, 101, Some("too-deep")).await, + Err(MegaError::BadRequest(_)) + )); +} + +async fn fixture() -> (TempDir, GitDbStorage) { + let dir = TempDir::new().unwrap(); + let connection = jupiter::tests::test_db_connection(dir.path()).await; + // Do not depend on feature unification to migrate this test database. + jupiter_migrate::apply_migrations(&connection, true) + .await + .unwrap(); + let storage = GitDbStorage { + base: BaseStorage::new(Arc::new(connection)), + }; + let now = chrono::Utc::now().naive_utc(); + for id in [101, 102] { + storage + .save_git_repo(git_repo::Model { + id, + repo_path: format!("/third-party/r{id}"), + repo_name: format!("r{id}"), + created_at: now, + updated_at: now, + }) + .await + .unwrap(); + } + (dir, storage) +} + +async fn commit(storage: &GitDbStorage, repo_id: i64, content: &str) -> (Commit, Tree) { + let tree = Tree::from_tree_items(vec![TreeItem { + mode: TreeItemMode::Blob, + id: Blob::from_content(content).id, + name: "file.txt".into(), + }]) + .unwrap(); + let commit = Commit::from_tree_id(tree.id, vec![], content); + let mut tree_model = tree.clone().into_git_model(EntryMeta::new()); + tree_model.repo_id = repo_id; + git_tree::Entity::insert(tree_model.into_active_model()) + .exec(storage.get_connection()) + .await + .unwrap(); + let mut commit_model = commit.clone().into_git_model(EntryMeta::new()); + commit_model.repo_id = repo_id; + git_commit::Entity::insert(commit_model.into_active_model()) + .exec(storage.get_connection()) + .await + .unwrap(); + (commit, tree) +} + +async fn reference(storage: &GitDbStorage, repo_id: i64, name: &str, oid: &str, default: bool) { + let now = chrono::Utc::now().naive_utc(); + storage + .save_ref( + repo_id, + import_refs::Model { + id: common::utils::generate_id(), + repo_id, + ref_name: name.into(), + ref_git_id: oid.into(), + ref_type: if name.starts_with("refs/tags/") { + RefTypeEnum::Tag + } else { + RefTypeEnum::Branch + }, + default_branch: default, + created_at: now, + updated_at: now, + }, + ) + .await + .unwrap(); +} + +async fn tag(storage: &GitDbStorage, id: &str, target: &str, kind: &str) { + storage + .insert_tag(git_tag::Model { + id: common::utils::generate_id(), + repo_id: 101, + tag_id: id.into(), + object_id: target.into(), + object_type: kind.into(), + tag_name: id.into(), + tagger: "fixture 0 +0000".into(), + message: "fixture".into(), + created_at: chrono::Utc::now().naive_utc(), + pack_id: String::new(), + pack_offset: 0, + }) + .await + .unwrap(); +} + +#[tokio::test] +async fn fixed_import_commit_and_resolved_ref_survive_branch_advance() { + let (_dir, storage) = fixture().await; + let (a, tree_a) = commit(&storage, 101, "A").await; + let (b, tree_b) = commit(&storage, 101, "B").await; + reference(&storage, 101, "refs/heads/main", &a.id.to_string(), true).await; + let pinned = resolve_import_commit(&storage, 101, Some("refs/heads/main")) + .await + .unwrap(); + storage + .update_ref(101, "refs/heads/main", &b.id.to_string()) + .await + .unwrap(); + assert_eq!( + read_import_root(&storage, &pinned).await.unwrap().id, + tree_a.id + ); + let explicit_a = resolve_import_commit(&storage, 101, Some(&a.id.to_string())) + .await + .unwrap(); + assert_eq!( + read_import_root(&storage, &explicit_a).await.unwrap().id, + tree_a.id + ); + let latest = resolve_import_commit(&storage, 101, None).await.unwrap(); + assert_eq!( + read_import_root(&storage, &latest).await.unwrap().id, + tree_b.id + ); +} + +#[tokio::test] +async fn missing_or_foreign_revision_never_falls_back_to_default() { + let (_dir, storage) = fixture().await; + let (a, _) = commit(&storage, 101, "A").await; + let (foreign, _) = commit(&storage, 102, "foreign").await; + reference(&storage, 101, "refs/heads/main", &a.id.to_string(), true).await; + for selector in [ + "missing-tag".to_owned(), + foreign.id.to_string(), + "0".repeat(40), + ] { + assert!(matches!( + resolve_import_commit(&storage, 101, Some(&selector)).await, + Err(MegaError::NotFound(_)) + )); + } +} + +#[tokio::test] +async fn fully_qualified_branch_and_legacy_tag_are_unambiguous() { + let (_dir, storage) = fixture().await; + let (a, _) = commit(&storage, 101, "A").await; + let (b, _) = commit(&storage, 101, "B").await; + reference(&storage, 101, "refs/heads/release", &b.id.to_string(), true).await; + reference(&storage, 101, "refs/tags/release", &a.id.to_string(), false).await; + for selector in ["release", "refs/tags/release"] { + assert_eq!( + resolve_import_commit(&storage, 101, Some(selector)) + .await + .unwrap() + .commit_oid, + a.id.to_string() + ); + } + assert_eq!( + resolve_import_commit(&storage, 101, Some("refs/heads/release")) + .await + .unwrap() + .commit_oid, + b.id.to_string() + ); +} + +#[tokio::test] +async fn annotated_tags_peel_only_from_a_tag_ref_and_never_follow_a_moved_tag() { + let (_dir, storage) = fixture().await; + let (a, _) = commit(&storage, 101, "A").await; + let (b, _) = commit(&storage, 101, "B").await; + let tag_id = "1".repeat(40); + tag(&storage, &tag_id, &a.id.to_string(), "commit").await; + reference(&storage, 101, "refs/tags/v1", &tag_id, false).await; + let pinned = resolve_import_commit(&storage, 101, Some("refs/tags/v1")) + .await + .unwrap(); + assert_eq!(pinned.commit_oid, a.id.to_string()); + assert!(matches!( + resolve_import_commit(&storage, 101, Some(&tag_id)).await, + Err(MegaError::NotFound(_)) + )); + storage + .update_ref(101, "refs/tags/v1", &b.id.to_string()) + .await + .unwrap(); + assert_eq!( + resolve_import_commit(&storage, 101, Some("v1")) + .await + .unwrap() + .commit_oid, + b.id.to_string() + ); + assert_eq!(pinned.commit_oid, a.id.to_string()); +} + +#[tokio::test] +async fn cyclic_and_non_commit_annotated_tags_fail_explicitly() { + let (_dir, storage) = fixture().await; + let first = "1".repeat(40); + let second = "2".repeat(40); + tag(&storage, &first, &second, "tag").await; + tag(&storage, &second, &first, "tag").await; + reference(&storage, 101, "refs/tags/cycle", &first, false).await; + assert!(matches!( + resolve_import_commit(&storage, 101, Some("cycle")).await, + Err(MegaError::BadRequest(_)) + )); + let third = "3".repeat(40); + tag(&storage, &third, &"4".repeat(40), "tree").await; + reference(&storage, 101, "refs/tags/tree", &third, false).await; + assert!(matches!( + resolve_import_commit(&storage, 101, Some("tree")).await, + Err(MegaError::BadRequest(_)) + )); +} + +#[tokio::test] +async fn absent_and_ambiguous_defaults_fail_without_panicking() { + let (_dir, storage) = fixture().await; + assert!(matches!( + resolve_import_commit(&storage, 101, None).await, + Err(MegaError::NotFound(_)) + )); + let (a, _) = commit(&storage, 101, "A").await; + reference(&storage, 101, "refs/heads/main", &a.id.to_string(), true).await; + reference(&storage, 101, "refs/heads/other", &a.id.to_string(), true).await; + assert!(matches!( + resolve_import_commit(&storage, 101, None).await, + Err(MegaError::Conflict(_)) + )); + // A valid explicit commit does not depend on an ambiguous default. + assert_eq!( + resolve_import_commit(&storage, 101, Some(&a.id.to_string())) + .await + .unwrap() + .commit_oid, + a.id.to_string() + ); +} diff --git a/ceres/src/model/mod.rs b/ceres/src/model/mod.rs index 1b067fea6..93878c622 100644 --- a/ceres/src/model/mod.rs +++ b/ceres/src/model/mod.rs @@ -13,10 +13,12 @@ pub mod group; pub mod issue; pub mod label; pub mod merge_queue; +pub mod namespace; pub mod note; pub mod notification; pub mod orion_runner; pub mod serde_snowflake; +pub mod snapshot; pub mod tag; pub mod third_party; pub mod user; diff --git a/ceres/src/model/namespace.rs b/ceres/src/model/namespace.rs new file mode 100644 index 000000000..ada73285b --- /dev/null +++ b/ceres/src/model/namespace.rs @@ -0,0 +1,323 @@ +//! Immutable namespace identity codec. Structural validity is not publication, +//! source authorization, scope attestation or an object-retention lease. + +use serde::{Deserialize, Serialize}; + +use super::snapshot::{ + IdentityError, MAX_PATH_BYTES, ManifestDigest, ObjectFormat, ObjectId, RelativePath, RepoPath, + SourceId, SourceSnapshot, +}; + +pub const MAX_MANIFEST_BYTES: usize = 16 * 1024; +const BINDING_DOMAIN: &[u8] = b"mega.namespace-binding.v1\0"; +const VIEW_DOMAIN: &[u8] = b"mega.namespace-view.v1\0"; + +/// Distinct from a source UUID even though both use the same UUID syntax. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(transparent)] +pub struct InstanceId(SourceId); + +impl InstanceId { + pub fn new(value: impl Into) -> Result { + SourceId::new(value).map(Self) + } + pub fn as_str(&self) -> &str { + self.0.as_str() + } +} + +/// Explicit data, never inferred from a numeric directory name. Encoding both +/// policies does not choose the deployment's release-directory policy. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum BindingPolicy { + Mutable, + ImmutableRelease, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum MaterializationPolicy { + GitRawV1, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(try_from = "BindingFields", into = "BindingFields")] +pub struct NamespaceBinding(BindingFields); + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct BindingFields { + mount_path: RepoPath, + source_snapshot: SourceSnapshot, + source_subpath: RelativePath, + policy: BindingPolicy, +} + +impl TryFrom for NamespaceBinding { + type Error = IdentityError; + fn try_from(fields: BindingFields) -> Result { + // A binding may expose a subtree of an attested source. It must still + // have a representable absolute source path for membership proofs. + let scope = fields.source_snapshot.scope_path.as_str(); + let subpath = fields.source_subpath.as_str(); + let length = scope.len() + usize::from(scope != "/" && !subpath.is_empty()) + subpath.len(); + if length > MAX_PATH_BYTES { + return Err(IdentityError("binding source path exceeds v1 limit")); + } + Ok(Self(fields)) + } +} +impl From for BindingFields { + fn from(value: NamespaceBinding) -> Self { + value.0 + } +} + +impl NamespaceBinding { + pub fn new( + mount_path: RepoPath, + source_snapshot: SourceSnapshot, + source_subpath: RelativePath, + policy: BindingPolicy, + ) -> Result { + BindingFields { + mount_path, + source_snapshot, + source_subpath, + policy, + } + .try_into() + } + pub fn mount_path(&self) -> &RepoPath { + &self.0.mount_path + } + pub fn source_snapshot(&self) -> &SourceSnapshot { + &self.0.source_snapshot + } + pub fn source_subpath(&self) -> &RelativePath { + &self.0.source_subpath + } + pub fn policy(&self) -> BindingPolicy { + self.0.policy + } + + pub fn canonical_bytes(&self) -> Vec { + let mut out = BINDING_DOMAIN.to_vec(); + frame(&mut out, self.mount_path().as_str().as_bytes()); + frame(&mut out, &self.source_snapshot().canonical_bytes()); + frame(&mut out, self.source_subpath().as_str().as_bytes()); + out.push(match self.policy() { + BindingPolicy::Mutable => 1, + BindingPolicy::ImmutableRelease => 2, + }); + out + } + + pub fn from_canonical_bytes(bytes: &[u8]) -> Result { + let mut r = Reader::new(bytes, BINDING_DOMAIN)?; + let mount = RepoPath::new(r.text()?)?; + let source = read_source(r.field()?)?; + let subpath = RelativePath::new(r.text()?)?; + let policy = match r.byte()? { + 1 => BindingPolicy::Mutable, + 2 => BindingPolicy::ImmutableRelease, + _ => return Err(IdentityError("unknown binding policy")), + }; + r.finish()?; + Self::new(mount, source, subpath, policy) + } + + pub fn id(&self) -> ManifestDigest { + hash_bytes(&self.canonical_bytes()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(try_from = "ViewFields", into = "ViewFields")] +pub struct NamespaceView(ViewFields); + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct ViewFields { + schema_version: u16, + instance_id: InstanceId, + native: SourceSnapshot, + bindings_root: ManifestDigest, + overrides_root: Option, + materialization_policy: MaterializationPolicy, +} + +impl TryFrom for NamespaceView { + type Error = IdentityError; + fn try_from(fields: ViewFields) -> Result { + if fields.schema_version != 1 { + return Err(IdentityError("unknown namespace view schema")); + } + if fields.native.scope_path.as_str() != "/" { + return Err(IdentityError( + "namespace native snapshot must cover root scope", + )); + } + Ok(Self(fields)) + } +} +impl From for ViewFields { + fn from(value: NamespaceView) -> Self { + value.0 + } +} + +impl NamespaceView { + pub fn new( + instance_id: InstanceId, + native: SourceSnapshot, + bindings_root: ManifestDigest, + overrides_root: Option, + materialization_policy: MaterializationPolicy, + ) -> Result { + ViewFields { + schema_version: 1, + instance_id, + native, + bindings_root, + overrides_root, + materialization_policy, + } + .try_into() + } + pub fn instance_id(&self) -> &InstanceId { + &self.0.instance_id + } + pub fn native(&self) -> &SourceSnapshot { + &self.0.native + } + pub fn bindings_root(&self) -> &ManifestDigest { + &self.0.bindings_root + } + pub fn overrides_root(&self) -> Option<&ManifestDigest> { + self.0.overrides_root.as_ref() + } + pub fn materialization_policy(&self) -> MaterializationPolicy { + self.0.materialization_policy + } + + pub fn canonical_bytes(&self) -> Vec { + let mut out = VIEW_DOMAIN.to_vec(); + out.extend_from_slice(&1u16.to_be_bytes()); + frame(&mut out, self.instance_id().as_str().as_bytes()); + frame(&mut out, &self.native().canonical_bytes()); + out.extend_from_slice(&raw_digest(self.bindings_root())); + match self.overrides_root() { + None => out.push(0), + Some(root) => { + out.push(1); + out.extend_from_slice(&raw_digest(root)); + } + } + out.push(match self.materialization_policy() { + MaterializationPolicy::GitRawV1 => 1, + }); + out + } + + pub fn from_canonical_bytes(bytes: &[u8]) -> Result { + let mut r = Reader::new(bytes, VIEW_DOMAIN)?; + if r.take(2)? != [0, 1] { + return Err(IdentityError("unknown namespace view schema")); + } + let instance = InstanceId::new(r.text()?)?; + let native = read_source(r.field()?)?; + let bindings = r.digest()?; + let overrides = match r.byte()? { + 0 => None, + 1 => Some(r.digest()?), + _ => return Err(IdentityError("invalid overrides presence tag")), + }; + let policy = match r.byte()? { + 1 => MaterializationPolicy::GitRawV1, + _ => return Err(IdentityError("unknown materialization policy")), + }; + r.finish()?; + Self::new(instance, native, bindings, overrides, policy) + } + + pub fn id(&self) -> ManifestDigest { + hash_bytes(&self.canonical_bytes()) + } +} + +fn frame(out: &mut Vec, value: &[u8]) { + out.extend_from_slice(&(value.len() as u32).to_be_bytes()); + out.extend_from_slice(value); +} +fn raw_digest(digest: &ManifestDigest) -> Vec { + hex::decode(&digest.as_str()[7..]).expect("validated digest") +} +fn hash_bytes(bytes: &[u8]) -> ManifestDigest { + use sha2::Digest; + let hex_digest = hex::encode(sha2::Sha256::digest(bytes)); + ManifestDigest::new(format!("sha256:{hex_digest}")).expect("SHA-256 digest") +} +struct Reader<'a>(&'a [u8]); +impl<'a> Reader<'a> { + fn new(bytes: &'a [u8], domain: &[u8]) -> Result { + if bytes.len() > MAX_MANIFEST_BYTES { + return Err(IdentityError("namespace manifest exceeds byte limit")); + } + bytes + .strip_prefix(domain) + .map(Self) + .ok_or(IdentityError("invalid manifest domain")) + } + fn take(&mut self, len: usize) -> Result<&'a [u8], IdentityError> { + if len > self.0.len() { + return Err(IdentityError("truncated manifest")); + } + let (head, tail) = self.0.split_at(len); + self.0 = tail; + Ok(head) + } + fn byte(&mut self) -> Result { + Ok(self.take(1)?[0]) + } + fn field(&mut self) -> Result<&'a [u8], IdentityError> { + let size = u32::from_be_bytes(self.take(4)?.try_into().expect("four bytes")) as usize; + self.take(size) + } + fn text(&mut self) -> Result<&'a str, IdentityError> { + std::str::from_utf8(self.field()?).map_err(|_| IdentityError("non-UTF-8 manifest field")) + } + fn digest(&mut self) -> Result { + ManifestDigest::new(format!("sha256:{}", hex::encode(self.take(32)?))) + } + fn finish(self) -> Result<(), IdentityError> { + if self.0.is_empty() { + Ok(()) + } else { + Err(IdentityError("trailing manifest bytes")) + } + } +} +fn read_source(bytes: &[u8]) -> Result { + let mut r = Reader::new(bytes, b"mega.source-snapshot.v1\0")?; + let source_id = SourceId::new(r.text()?)?; + let scope_path = RepoPath::new(r.text()?)?; + let object_format = match r.text()? { + "sha1" => ObjectFormat::Sha1, + _ => return Err(IdentityError("unknown source object format")), + }; + let commit_oid = ObjectId::new(r.text()?)?; + let root_tree_oid = ObjectId::new(r.text()?)?; + r.finish()?; + Ok(SourceSnapshot { + source_id, + scope_path, + object_format, + commit_oid, + root_tree_oid, + }) +} + +#[cfg(test)] +mod tests; diff --git a/ceres/src/model/namespace/tests.rs b/ceres/src/model/namespace/tests.rs new file mode 100644 index 000000000..4dbc81751 --- /dev/null +++ b/ceres/src/model/namespace/tests.rs @@ -0,0 +1,212 @@ +use super::*; + +fn vectors() -> serde_json::Value { + serde_json::from_str(include_str!( + "../../../tests/fixtures/snapshot/namespace-v1.json" + )) + .unwrap() +} +fn binding() -> NamespaceBinding { + serde_json::from_value(vectors()["bindings"][0]["binding"].clone()).unwrap() +} +fn view() -> NamespaceView { + serde_json::from_value(vectors()["views"][0]["view"].clone()).unwrap() +} + +#[test] +fn snapshot_namespace_vectors_match_independent_dotnet_and_json_roundtrip() { + for vector in vectors()["bindings"].as_array().unwrap() { + let value: NamespaceBinding = serde_json::from_value(vector["binding"].clone()).unwrap(); + let bytes = hex::decode(vector["canonical_hex"].as_str().unwrap()).unwrap(); + assert_eq!(value.canonical_bytes(), bytes); + assert_eq!(value.id().as_str(), vector["digest"].as_str().unwrap()); + assert_eq!( + NamespaceBinding::from_canonical_bytes(&bytes).unwrap(), + value + ); + assert_eq!(serde_json::to_value(value).unwrap(), vector["binding"]); + } + for vector in vectors()["views"].as_array().unwrap() { + let value: NamespaceView = serde_json::from_value(vector["view"].clone()).unwrap(); + let bytes = hex::decode(vector["canonical_hex"].as_str().unwrap()).unwrap(); + assert_eq!(value.canonical_bytes(), bytes); + assert_eq!(value.id().as_str(), vector["digest"].as_str().unwrap()); + assert_eq!(NamespaceView::from_canonical_bytes(&bytes).unwrap(), value); + assert_eq!(serde_json::to_value(value).unwrap(), vector["view"]); + } +} + +#[test] +fn snapshot_namespace_provenance_routing_policy_and_instance_affect_identity() { + let base = view(); + let other_instance = NamespaceView::new( + InstanceId::new("99999999-9999-4999-8999-999999999999").unwrap(), + base.native().clone(), + base.bindings_root().clone(), + None, + base.materialization_policy(), + ) + .unwrap(); + assert_ne!(base.id(), other_instance.id()); + let mut native = base.native().clone(); + native.commit_oid = ObjectId::new("f".repeat(40)).unwrap(); + let other_commit = NamespaceView::new( + base.instance_id().clone(), + native, + base.bindings_root().clone(), + None, + base.materialization_policy(), + ) + .unwrap(); + assert_eq!( + base.native().root_tree_oid, + other_commit.native().root_tree_oid + ); + assert_ne!(base.id(), other_commit.id()); + let other_bindings = NamespaceView::new( + base.instance_id().clone(), + base.native().clone(), + hash_bytes(b"different routing"), + None, + base.materialization_policy(), + ) + .unwrap(); + assert_ne!(base.id(), other_bindings.id()); + let base_binding = binding(); + let other_policy = NamespaceBinding::new( + base_binding.mount_path().clone(), + base_binding.source_snapshot().clone(), + base_binding.source_subpath().clone(), + BindingPolicy::ImmutableRelease, + ) + .unwrap(); + assert_ne!(base_binding.id(), other_policy.id()); + let moved = NamespaceBinding::new( + RepoPath::new("/other").unwrap(), + base_binding.source_snapshot().clone(), + base_binding.source_subpath().clone(), + base_binding.policy(), + ) + .unwrap(); + assert_ne!(base_binding.id(), moved.id()); + let subpath = NamespaceBinding::new( + base_binding.mount_path().clone(), + base_binding.source_snapshot().clone(), + RelativePath::new("other").unwrap(), + base_binding.policy(), + ) + .unwrap(); + assert_ne!(base_binding.id(), subpath.id()); +} + +#[test] +fn snapshot_namespace_json_cannot_bypass_schema_scope_or_unknown_field_checks() { + let raw = vectors()["views"][0]["view"].clone(); + for (field, value) in [ + ("schema_version", serde_json::json!(2)), + ( + "instance_id", + serde_json::json!("00000000-0000-0000-0000-000000000000"), + ), + ( + "materialization_policy", + serde_json::json!("hydrate_everything"), + ), + ("lease", serde_json::json!("not identity")), + ("publication_seq", serde_json::json!(1)), + ] { + let mut changed = raw.clone(); + changed[field] = value; + assert!( + serde_json::from_value::(changed).is_err(), + "{field}" + ); + } + let mut scoped = raw; + scoped["native"]["scope_path"] = serde_json::json!("/child"); + assert!(serde_json::from_value::(scoped).is_err()); + let raw = vectors()["bindings"][0]["binding"].clone(); + for (field, value) in [ + ("mount_path", serde_json::json!("/deps//bad")), + ("source_subpath", serde_json::json!("../escape")), + ("policy", serde_json::json!("guess_from_path")), + ("ref_name", serde_json::json!("refs/heads/main")), + ] { + let mut changed = raw.clone(); + changed[field] = value; + assert!( + serde_json::from_value::(changed).is_err(), + "{field}" + ); + } +} + +#[test] +fn snapshot_namespace_codec_rejects_truncation_oversize_unknown_tags_and_domains() { + let binding = binding().canonical_bytes(); + let view = view().canonical_bytes(); + for end in 0..binding.len() { + assert!(NamespaceBinding::from_canonical_bytes(&binding[..end]).is_err()); + } + for end in 0..view.len() { + assert!(NamespaceView::from_canonical_bytes(&view[..end]).is_err()); + } + assert!(NamespaceBinding::from_canonical_bytes(&view).is_err()); + assert!(NamespaceView::from_canonical_bytes(&binding).is_err()); + for original in [&binding, &view] { + let mut extra = original.clone(); + extra.push(0); + assert!(NamespaceBinding::from_canonical_bytes(&extra).is_err()); + assert!(NamespaceView::from_canonical_bytes(&extra).is_err()); + } + let mut bad = binding.clone(); + *bad.last_mut().unwrap() = 0; + assert!(NamespaceBinding::from_canonical_bytes(&bad).is_err()); + let mut bad = view.clone(); + *bad.last_mut().unwrap() = 99; + assert!(NamespaceView::from_canonical_bytes(&bad).is_err()); + let mut bad = view.clone(); + bad[view.len() - 2] = 3; + assert!(NamespaceView::from_canonical_bytes(&bad).is_err()); + let mut bad = view; + bad[VIEW_DOMAIN.len() + 1] = 2; + assert!(NamespaceView::from_canonical_bytes(&bad).is_err()); + let mut bad = binding; + bad[BINDING_DOMAIN.len()..BINDING_DOMAIN.len() + 4].copy_from_slice(&u32::MAX.to_be_bytes()); + assert!(NamespaceBinding::from_canonical_bytes(&bad).is_err()); + let huge = vec![0; MAX_MANIFEST_BYTES + 1]; + assert!(NamespaceView::from_canonical_bytes(&huge).is_err()); + assert!(NamespaceBinding::from_canonical_bytes(&huge).is_err()); +} + +#[test] +fn snapshot_namespace_maximum_paths_fit_bounded_manifests_and_proofs() { + let path = format!("/{}", vec!["x".repeat(255); 16].join("/")); + assert_eq!(path.len(), MAX_PATH_BYTES); + let mut native = view().native().clone(); + native.scope_path = RepoPath::new(&path).unwrap(); + let maximal = NamespaceBinding::new( + RepoPath::new(&path).unwrap(), + native.clone(), + RelativePath::new("").unwrap(), + BindingPolicy::Mutable, + ) + .unwrap(); + assert!(maximal.canonical_bytes().len() < MAX_MANIFEST_BYTES); + assert_eq!( + NamespaceBinding::from_canonical_bytes(&maximal.canonical_bytes()).unwrap(), + maximal + ); + assert!( + NamespaceBinding::new( + RepoPath::new("/deps").unwrap(), + native, + RelativePath::new("x").unwrap(), + BindingPolicy::Mutable, + ) + .is_err() + ); + let mut raw = serde_json::to_value(maximal).unwrap(); + raw["source_subpath"] = serde_json::json!("x"); + assert!(serde_json::from_value::(raw).is_err()); +} diff --git a/ceres/src/model/snapshot.rs b/ceres/src/model/snapshot.rs new file mode 100644 index 000000000..6bdbf6366 --- /dev/null +++ b/ceres/src/model/snapshot.rs @@ -0,0 +1,367 @@ +//! Version-one source identities shared with the Mega snapshot contract. +//! +//! The JSON form is a wire representation, not the bytes to hash. IDs use the +//! explicitly framed encoding below; moving refs and leases are not identity. + +use std::{fmt, str::FromStr}; + +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct IdentityError(pub &'static str); + +impl fmt::Display for IdentityError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.0) + } +} +impl std::error::Error for IdentityError {} + +macro_rules! validated_string { + ($name:ident, $validator:ident, $message:literal) => { + #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] + #[serde(try_from = "String", into = "String")] + pub struct $name(String); + + impl $name { + pub fn new(value: impl Into) -> Result { + let value = value.into(); + if !$validator(&value) { + return Err(IdentityError($message)); + } + Ok(Self(value)) + } + pub fn as_str(&self) -> &str { + &self.0 + } + } + impl TryFrom for $name { + type Error = IdentityError; + fn try_from(value: String) -> Result { + Self::new(value) + } + } + impl From<$name> for String { + fn from(value: $name) -> Self { + value.0 + } + } + impl FromStr for $name { + type Err = IdentityError; + fn from_str(value: &str) -> Result { + Self::new(value) + } + } + impl fmt::Display for $name { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } + } + }; +} + +fn valid_source(value: &str) -> bool { + uuid::Uuid::parse_str(value) + .map(|id| !id.is_nil() && id.to_string() == value) + .unwrap_or(false) +} + +fn lowercase_hex(value: &str, len: usize) -> bool { + value.len() == len + && value + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} + +fn valid_oid(value: &str) -> bool { + lowercase_hex(value, 40) +} + +fn valid_digest(value: &str) -> bool { + value + .strip_prefix("sha256:") + .is_some_and(|hex| lowercase_hex(hex, 64)) +} + +/// Limits are byte lengths, matching the v1 Linux projection contract. +pub const MAX_PATH_BYTES: usize = 4096; +pub const MAX_COMPONENT_BYTES: usize = 255; + +fn valid_components(value: &str) -> bool { + value.split('/').all(|part| { + !part.is_empty() + && part != "." + && part != ".." + && part.len() <= MAX_COMPONENT_BYTES + && !part.contains('\0') + }) +} + +fn valid_absolute_path(value: &str) -> bool { + value == "/" + || (value.len() <= MAX_PATH_BYTES && value.strip_prefix('/').is_some_and(valid_components)) +} + +fn valid_relative_path(value: &str) -> bool { + value.is_empty() || (value.len() <= MAX_PATH_BYTES && valid_components(value)) +} + +fn valid_ref(value: &str) -> bool { + (value.starts_with("refs/heads/") || value.starts_with("refs/tags/")) + && value.len() <= 1024 + && !value.ends_with('.') + && !value.contains("..") + && !value.contains("@{") + && !value + .bytes() + .any(|b| b <= b' ' || b == 127 || b"~^:?*[\\".contains(&b)) + && value + .split('/') + .all(|part| !part.is_empty() && !part.starts_with('.') && !part.ends_with(".lock")) +} + +validated_string!( + SourceId, + valid_source, + "source ID must be a non-nil canonical UUID" +); +validated_string!( + ObjectId, + valid_oid, + "v1 Git object ID must be 40 lowercase hexadecimal digits" +); +validated_string!( + ManifestDigest, + valid_digest, + "digest must be sha256 followed by 64 lowercase hexadecimal digits" +); +validated_string!( + RepoPath, + valid_absolute_path, + "path must be absolute, canonical UTF-8 and within v1 byte limits" +); +validated_string!( + RelativePath, + valid_relative_path, + "relative path must be canonical UTF-8 and within v1 byte limits" +); +validated_string!( + RefName, + valid_ref, + "ref must be a canonical fully qualified branch or tag" +); + +impl RepoPath { + /// Component-aware containment, not a raw starts_with check. + pub fn relative_to(&self, scope: &RepoPath) -> Option { + let relative = if scope.as_str() == "/" { + self.as_str().strip_prefix('/')? + } else if self == scope { + "" + } else { + self.as_str() + .strip_prefix(scope.as_str())? + .strip_prefix('/')? + }; + RelativePath::new(relative).ok() + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum ObjectFormat { + Sha1, +} + +impl ObjectFormat { + pub fn as_str(self) -> &'static str { + match self { + Self::Sha1 => "sha1", + } + } +} + +/// Structural validity is not a server attestation: the resolver must also +/// prove the commit/tree/scope relationship and enforce authorization. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SourceSnapshot { + pub source_id: SourceId, + pub scope_path: RepoPath, + pub object_format: ObjectFormat, + pub commit_oid: ObjectId, + pub root_tree_oid: ObjectId, +} + +impl SourceSnapshot { + /// Domain bytes, then five UTF-8 fields with unsigned big-endian u32 lengths. + /// Field order is part of v1. There are no optional or floating-point fields. + pub fn canonical_bytes(&self) -> Vec { + let mut bytes = b"mega.source-snapshot.v1\0".to_vec(); + for field in [ + self.source_id.as_str(), + self.scope_path.as_str(), + self.object_format.as_str(), + self.commit_oid.as_str(), + self.root_tree_oid.as_str(), + ] { + // All fields are validated and bounded well below u32::MAX. + bytes.extend_from_slice(&(field.len() as u32).to_be_bytes()); + bytes.extend_from_slice(field.as_bytes()); + } + bytes + } + + /// Provenance identity. This is NOT a namespace view or a projection key: + /// two commits with the same tree may still share a verified object cache. + pub fn id(&self) -> ManifestDigest { + let bytes = self.canonical_bytes(); + let digest = { + use sha2::Digest; + hex::encode(sha2::Sha256::digest(&bytes)) + }; + ManifestDigest(format!("sha256:{digest}")) + } +} + +/// Only used before resolving. Immutable readers never retain this selector. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum SourceSelector { + SourceCommit { + source_id: SourceId, + scope_path: RepoPath, + commit_oid: ObjectId, + }, + SourceRef { + source_id: SourceId, + scope_path: RepoPath, + ref_name: RefName, + }, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[derive(Deserialize)] + struct Vector { + source: SourceSnapshot, + canonical_hex: String, + source_id_digest: ManifestDigest, + } + + #[test] + fn source_golden_vectors_match_independent_encoding() { + let vectors: Vec = + serde_json::from_str(include_str!("../../tests/fixtures/snapshot/source-v1.json")) + .unwrap(); + for vector in vectors { + assert_eq!( + hex::encode(vector.source.canonical_bytes()), + vector.canonical_hex + ); + assert_eq!(vector.source.id(), vector.source_id_digest); + let json = serde_json::to_string(&vector.source).unwrap(); + assert_eq!( + serde_json::from_str::(&json).unwrap(), + vector.source + ); + } + } + + #[test] + fn paths_preserve_names_and_enforce_component_boundaries() { + let scope = RepoPath::new("/project/a").unwrap(); + assert_eq!( + RepoPath::new("/project/a/src/a+b.rs") + .unwrap() + .relative_to(&scope) + .unwrap() + .as_str(), + "src/a+b.rs" + ); + assert!( + RepoPath::new("/project/ab") + .unwrap() + .relative_to(&scope) + .is_none() + ); + assert_eq!(scope.relative_to(&scope).unwrap().as_str(), ""); + let unicode = RepoPath::new("/第三方/e\u{301}").unwrap(); + assert_eq!(unicode.as_str(), "/第三方/e\u{301}"); + for invalid in [ + "", + "project/a", + "//a", + "/a/", + "/a//b", + "/a/./b", + "/a/../b", + "/a\0b", + ] { + assert!(RepoPath::new(invalid).is_err(), "{invalid:?}"); + } + assert!(RepoPath::new(format!("/{}", "a".repeat(256))).is_err()); + assert!(RelativePath::new("/absolute").is_err()); + assert!(RelativePath::new("../outside").is_err()); + } + + #[test] + fn deserialization_cannot_bypass_identity_validation() { + for invalid in [ + "\"BAD\"", + "\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"", + "\"0000000000000000000000000000000000000000000000000000000000000000\"", + ] { + assert!(serde_json::from_str::(invalid).is_err()); + } + assert!(SourceId::new("00000000-0000-0000-0000-000000000000").is_err()); + assert!(SourceId::new("https://example.test/repo").is_err()); + assert!(serde_json::from_str::("\"sha256\"").is_err()); + assert!(ManifestDigest::new(format!("sha256:{}", "A".repeat(64))).is_err()); + } + + #[test] + fn symbolic_refs_are_typed_and_fully_qualified() { + for valid in [ + "refs/heads/main", + "refs/tags/v1.2.3+build", + "refs/heads/团队/分支", + ] { + assert!(RefName::new(valid).is_ok(), "{valid}"); + } + for invalid in [ + "main", + "HEAD", + "refs/cl/123", + "refs/heads/", + "refs/heads/a..b", + "refs/tags/.x", + "refs/tags/x.lock", + "refs/heads/a@{b", + "refs/heads/a//b", + "refs/heads/a?b", + "refs/heads/a\\b", + ] { + assert!(RefName::new(invalid).is_err(), "{invalid}"); + } + } + + #[test] + fn provenance_includes_scope_source_and_commit_even_when_tree_is_equal() { + let vectors: Vec = + serde_json::from_str(include_str!("../../tests/fixtures/snapshot/source-v1.json")) + .unwrap(); + let source = vectors.into_iter().next().unwrap().source; + let mut other = source.clone(); + other.commit_oid = ObjectId::new("2".repeat(40)).unwrap(); + assert_ne!(source.id(), other.id()); + other = source.clone(); + other.scope_path = RepoPath::new("/different").unwrap(); + assert_ne!(source.id(), other.id()); + other = source.clone(); + other.source_id = SourceId::new("22222222-2222-4222-8222-222222222222").unwrap(); + assert_ne!(source.id(), other.id()); + } +} diff --git a/ceres/tests/fixtures/snapshot/namespace-radix-v1.json b/ceres/tests/fixtures/snapshot/namespace-radix-v1.json new file mode 100644 index 000000000..3b7b2ed02 --- /dev/null +++ b/ceres/tests/fixtures/snapshot/namespace-radix-v1.json @@ -0,0 +1,17 @@ +[ + { + "name": "empty", + "canonical_hex": "6d6567612e6e616d6573706163652d72616469782e7631000000000000", + "digest": "sha256:18946486089198dfa8eeb70fa90e04b137c579dc08ae1e6f8bceafc0d35ef677" + }, + { + "name": "leaf-a", + "canonical_hex": "6d6567612e6e616d6573706163652d72616469782e763100000261000111111111111111111111111111111111111111111111111111111111111111110000", + "digest": "sha256:0daf3f5407c62015b13000cc10e525975b5893f08bdbbe2a919d63c8bdc18cae" + }, + { + "name": "branch-ab", + "canonical_hex": "6d6567612e6e616d6573706163652d72616469782e7631000000000002612222222222222222222222222222222222222222222222222222222222222222623333333333333333333333333333333333333333333333333333333333333333", + "digest": "sha256:6eff5588f487756adb7b0b1834929ae1f7800e0c15553cc54836b78b9501e74e" + } +] diff --git a/ceres/tests/fixtures/snapshot/namespace-v1-vectors.ps1 b/ceres/tests/fixtures/snapshot/namespace-v1-vectors.ps1 new file mode 100644 index 000000000..b68feb017 --- /dev/null +++ b/ceres/tests/fixtures/snapshot/namespace-v1-vectors.ps1 @@ -0,0 +1,69 @@ +# Independent .NET framing/SHA-256 oracle. Emits JSON only; never calls Rust. +# Run in PowerShell 7. Do not regenerate expected vectors using the codec under test. +$ErrorActionPreference = 'Stop' +function New-Bytes([string]$domain) { + $buffer = [System.Collections.Generic.List[byte]]::new() + $buffer.AddRange([System.Text.Encoding]::UTF8.GetBytes($domain)) + $buffer.Add(0) + return ,$buffer +} +function Add-Field($buffer, [byte[]]$bytes) { + $length = [System.BitConverter]::GetBytes([uint32]$bytes.Length) + if ([System.BitConverter]::IsLittleEndian) { [array]::Reverse($length) } + $buffer.AddRange($length) + $buffer.AddRange($bytes) +} +function Add-Text($buffer, [string]$value) { + Add-Field $buffer ([System.Text.Encoding]::UTF8.GetBytes($value)) +} +function Source-Bytes($source) { + $buffer = New-Bytes 'mega.source-snapshot.v1' + foreach ($field in @('source_id','scope_path','object_format','commit_oid','root_tree_oid')) { + Add-Text $buffer $source[$field] + } + return ,$buffer.ToArray() +} +function Digest([byte[]]$bytes) { + return 'sha256:' + [System.Convert]::ToHexString([System.Security.Cryptography.SHA256]::HashData($bytes)).ToLowerInvariant() +} +function Hex([byte[]]$bytes) { return [System.Convert]::ToHexString($bytes).ToLowerInvariant() } +$native = [ordered]@{ + source_id='11111111-1111-4111-8111-111111111111'; scope_path='/'; object_format='sha1' + commit_oid=('1' * 40); root_tree_oid='4b825dc642cb6eb9a060e54bf8d69288fbee4904' +} +$import = [ordered]@{ + source_id='33333333-3333-4333-8333-333333333333'; scope_path='/third-party/库+1'; object_format='sha1' + commit_oid=('a' * 40); root_tree_oid=('b' * 40) +} +$bindings = @() +foreach ($policy in @('mutable','immutable_release')) { + $binding = [ordered]@{ + mount_path='/deps/库+1'; source_snapshot=$import; source_subpath='src'; policy=$policy + } + $buffer = New-Bytes 'mega.namespace-binding.v1' + Add-Text $buffer $binding.mount_path + Add-Field $buffer (Source-Bytes $import) + Add-Text $buffer $binding.source_subpath + if ($policy -eq 'mutable') { $buffer.Add(1) } else { $buffer.Add(2) } + $bindings += [ordered]@{binding=$binding; canonical_hex=(Hex $buffer.ToArray()); digest=(Digest $buffer.ToArray())} +} +$empty = 'sha256:18946486089198dfa8eeb70fa90e04b137c579dc08ae1e6f8bceafc0d35ef677' +$views = @() +foreach ($overrides in @($null, ('sha256:' + ('c' * 64)))) { + $view = [ordered]@{ + schema_version=1; instance_id='22222222-2222-4222-8222-222222222222'; native=$native + bindings_root=$empty; overrides_root=$overrides; materialization_policy='git_raw_v1' + } + $buffer = New-Bytes 'mega.namespace-view.v1' + $buffer.AddRange([byte[]]@(0,1)) + Add-Text $buffer $view.instance_id + Add-Field $buffer (Source-Bytes $native) + $buffer.AddRange([System.Convert]::FromHexString($empty.Substring(7))) + if ($null -eq $overrides) { $buffer.Add(0) } else { + $buffer.Add(1) + $buffer.AddRange([System.Convert]::FromHexString($overrides.Substring(7))) + } + $buffer.Add(1) + $views += [ordered]@{view=$view; canonical_hex=(Hex $buffer.ToArray()); digest=(Digest $buffer.ToArray())} +} +[ordered]@{bindings=$bindings; views=$views} | ConvertTo-Json -Depth 10 diff --git a/ceres/tests/fixtures/snapshot/namespace-v1.json b/ceres/tests/fixtures/snapshot/namespace-v1.json new file mode 100644 index 000000000..0d84de567 --- /dev/null +++ b/ceres/tests/fixtures/snapshot/namespace-v1.json @@ -0,0 +1,74 @@ +{ + "bindings": [ + { + "binding": { + "mount_path": "/deps/库+1", + "source_snapshot": { + "source_id": "33333333-3333-4333-8333-333333333333", + "scope_path": "/third-party/库+1", + "object_format": "sha1", + "commit_oid": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "root_tree_oid": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + }, + "source_subpath": "src", + "policy": "mutable" + }, + "canonical_hex": "6d6567612e6e616d6573706163652d62696e64696e672e7631000000000b2f646570732fe5ba932b31000000b66d6567612e736f757263652d736e617073686f742e7631000000002433333333333333332d333333332d343333332d383333332d333333333333333333333333000000122f74686972642d70617274792fe5ba932b310000000473686131000000286161616161616161616161616161616161616161616161616161616161616161616161616161616100000028626262626262626262626262626262626262626262626262626262626262626262626262626262620000000373726301", + "digest": "sha256:adebe124b05761074c9460ed20426acf3023645e2bfa7e46b12239da68b14a88" + }, + { + "binding": { + "mount_path": "/deps/库+1", + "source_snapshot": { + "source_id": "33333333-3333-4333-8333-333333333333", + "scope_path": "/third-party/库+1", + "object_format": "sha1", + "commit_oid": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "root_tree_oid": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + }, + "source_subpath": "src", + "policy": "immutable_release" + }, + "canonical_hex": "6d6567612e6e616d6573706163652d62696e64696e672e7631000000000b2f646570732fe5ba932b31000000b66d6567612e736f757263652d736e617073686f742e7631000000002433333333333333332d333333332d343333332d383333332d333333333333333333333333000000122f74686972642d70617274792fe5ba932b310000000473686131000000286161616161616161616161616161616161616161616161616161616161616161616161616161616100000028626262626262626262626262626262626262626262626262626262626262626262626262626262620000000373726302", + "digest": "sha256:4f4263e0096171458b5bb5915497e20c74af2a39c01270ce591d629af63d5ae0" + } + ], + "views": [ + { + "view": { + "schema_version": 1, + "instance_id": "22222222-2222-4222-8222-222222222222", + "native": { + "source_id": "11111111-1111-4111-8111-111111111111", + "scope_path": "/", + "object_format": "sha1", + "commit_oid": "1111111111111111111111111111111111111111", + "root_tree_oid": "4b825dc642cb6eb9a060e54bf8d69288fbee4904" + }, + "bindings_root": "sha256:18946486089198dfa8eeb70fa90e04b137c579dc08ae1e6f8bceafc0d35ef677", + "overrides_root": null, + "materialization_policy": "git_raw_v1" + }, + "canonical_hex": "6d6567612e6e616d6573706163652d766965772e76310000010000002432323232323232322d323232322d343232322d383232322d323232323232323232323232000000a56d6567612e736f757263652d736e617073686f742e7631000000002431313131313131312d313131312d343131312d383131312d313131313131313131313131000000012f00000004736861310000002831313131313131313131313131313131313131313131313131313131313131313131313131313131000000283462383235646336343263623665623961303630653534626638643639323838666265653439303418946486089198dfa8eeb70fa90e04b137c579dc08ae1e6f8bceafc0d35ef6770001", + "digest": "sha256:3c8632afb308bf562973b3af517ae5d0a27c05651f3f7511f91e16d7ad8f1231" + }, + { + "view": { + "schema_version": 1, + "instance_id": "22222222-2222-4222-8222-222222222222", + "native": { + "source_id": "11111111-1111-4111-8111-111111111111", + "scope_path": "/", + "object_format": "sha1", + "commit_oid": "1111111111111111111111111111111111111111", + "root_tree_oid": "4b825dc642cb6eb9a060e54bf8d69288fbee4904" + }, + "bindings_root": "sha256:18946486089198dfa8eeb70fa90e04b137c579dc08ae1e6f8bceafc0d35ef677", + "overrides_root": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "materialization_policy": "git_raw_v1" + }, + "canonical_hex": "6d6567612e6e616d6573706163652d766965772e76310000010000002432323232323232322d323232322d343232322d383232322d323232323232323232323232000000a56d6567612e736f757263652d736e617073686f742e7631000000002431313131313131312d313131312d343131312d383131312d313131313131313131313131000000012f00000004736861310000002831313131313131313131313131313131313131313131313131313131313131313131313131313131000000283462383235646336343263623665623961303630653534626638643639323838666265653439303418946486089198dfa8eeb70fa90e04b137c579dc08ae1e6f8bceafc0d35ef67701cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc01", + "digest": "sha256:671360699c27ba18ae4de2b1288c90f5849b3e381170c33b83b370c42a660a4e" + } + ] +} diff --git a/ceres/tests/fixtures/snapshot/source-v1.json b/ceres/tests/fixtures/snapshot/source-v1.json new file mode 100644 index 000000000..e3cda4264 --- /dev/null +++ b/ceres/tests/fixtures/snapshot/source-v1.json @@ -0,0 +1,24 @@ +[ + { + "source": { + "source_id": "11111111-1111-4111-8111-111111111111", + "scope_path": "/project/a", + "object_format": "sha1", + "commit_oid": "1111111111111111111111111111111111111111", + "root_tree_oid": "4b825dc642cb6eb9a060e54bf8d69288fbee4904" + }, + "canonical_hex": "6d6567612e736f757263652d736e617073686f742e7631000000002431313131313131312d313131312d343131312d383131312d3131313131313131313131310000000a2f70726f6a6563742f61000000047368613100000028313131313131313131313131313131313131313131313131313131313131313131313131313131310000002834623832356463363432636236656239613036306535346266386436393238386662656534393034", + "source_id_digest": "sha256:6e3f8a7e41d3a9759bc05cbc1dab153ad27ba0e0ff494f7692392dbfd5a95451" + }, + { + "source": { + "source_id": "33333333-3333-4333-8333-333333333333", + "scope_path": "/third-party/库+1", + "object_format": "sha1", + "commit_oid": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "root_tree_oid": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + }, + "canonical_hex": "6d6567612e736f757263652d736e617073686f742e7631000000002433333333333333332d333333332d343333332d383333332d333333333333333333333333000000122f74686972642d70617274792fe5ba932b31000000047368613100000028616161616161616161616161616161616161616161616161616161616161616161616161616161610000002862626262626262626262626262626262626262626262626262626262626262626262626262626262", + "source_id_digest": "sha256:4e5857227f3b44e979c35aadcdc92835646a1eadae013d3e01ec07a08017dd74" + } +] diff --git a/docs/monorepo-versioning-design.md b/docs/monorepo-versioning-design.md new file mode 100644 index 000000000..29bdb201c --- /dev/null +++ b/docs/monorepo-versioning-design.md @@ -0,0 +1,393 @@ +# Mega 如何给整个 monorepo 定版本 + +## 30 秒说明 + +Mega 展示出来的目录并不只属于一个 Git 仓库。它既有主仓自己的目录，也会把其他仓库挂载到某些路径下。 + +因此，只记录主仓的 commit 不能代表整个 monorepo 的状态。构建进行到一半时，如果某个挂载仓库更新了，同一次构建就可能读到新旧混合的文件。 + +本设计只引入一个对用户可见的新东西：**monorepo 版本**。 + +一个 monorepo 版本会同时锁定： + +- 主仓使用哪个 commit； +- 每个挂载仓库使用哪个 commit； +- 这些仓库分别挂载到哪个目录。 + +Mega 负责生成和保存这个版本，ScorpioFS 只负责按指定版本把目录挂载出来。 + +~~~text +开发者 push / merge + ↓ +Mega 发布一个完整的 monorepo 版本 + ↓ +ScorpioFS 固定读取这个版本 + ↓ +同一个版本始终看到同一批文件 +~~~ + +## 现在会出什么问题 + +假设 Mega 中有以下目录： + +~~~text +/project/app 主仓中的代码 +/third-party/lib 从另一个 Git 仓库挂载进来的依赖 +~~~ + +构建开始时： + +- 主仓是 M10； +- 依赖仓是 L20。 + +构建先读取了 /project/app。几分钟后，依赖仓的默认分支更新为 L21，构建这时才读取 /third-party/lib。 + +最终，这次构建实际使用的是： + +~~~text +主仓 M10 + 依赖仓 L21 +~~~ + +但这个组合可能从来没有被测试或发布过。只固定主仓 M10 解决不了问题，因为 L20 和 L21 都不属于主仓。 + +我们希望构建拿到的是一个完整版本，例如： + +~~~text +版本 V100 +├── /project/app → 主仓 M10 +└── /third-party/lib → 依赖仓 L20 +~~~ + +即使依赖仓后来更新到 L21，读取 V100 仍然只能看到 L20。需要使用新依赖时，Mega 再发布 V101，而不是修改 V100。 + +## 一个完整例子：依赖仓从 L20 升级到 L21 + +如果只想快速理解设计，可以只看这一节。 + +### 1. 发布 V100 + +假设对外目录是： + +~~~text +/ +├── project/app/ 来自主仓 M10 +└── third-party/lib/ 来自依赖仓 L20 的 src/ 目录 +~~~ + +为了方便阅读，可以先把 V100 想成下面这份展开后的数据。M10、L20、MT10、LT20 是简写，分别表示 commit 和 tree： + +~~~json +{ + "version": "V100", + "native": { + "commit": "M10", + "tree": "MT10" + }, + "mounts": [ + { + "path": "/third-party/lib", + "source": { + "commit": "L20", + "tree": "LT20" + }, + "source_subpath": "src", + "policy": "mutable" + } + ] +} +~~~ + +这份数据表达了两件事： + +- /project/app/main.rs 必须从主仓 M10 的 MT10 中读取； +- /third-party/lib/parser.rs 必须从依赖仓 L20 的 LT20/src/parser.rs 中读取。 + +ScorpioFS 创建 workspace 时固定 V100。此后即使依赖仓的 main 分支移动，它也不会重新解析 main。 + +### 2. 依赖仓更新 + +现在开发者把依赖仓被选中的 main 分支从 L20 push 到 L21。Mega 在一次发布操作中同时完成： + +~~~text +依赖仓 main：L20 → L21 +monorepo latest：V100 → V101 +~~~ + +V101 展开后是： + +~~~json +{ + "version": "V101", + "native": { + "commit": "M10", + "tree": "MT10" + }, + "mounts": [ + { + "path": "/third-party/lib", + "source": { + "commit": "L21", + "tree": "LT21" + }, + "source_subpath": "src", + "policy": "mutable" + } + ] +} +~~~ + +两份版本的结果非常直接： + +| workspace 固定的版本 | /project/app | /third-party/lib | +| --- | --- | --- | +| V100 | M10 | L20 | +| V101 | M10 | L21 | + +V101 没有复制主仓文件，只是继续引用 M10。V100 也没有被改写，所以两个 workspace 可以同时工作。 + +### 3. 实现中实际保存的数据结构 + +上面的 mounts 数组是方便人阅读的展开形式。实际实现使用以下三个核心结构： + +~~~text +SourceSnapshot { + source_id, // 稳定的仓库身份 + scope_path, // 这份快照对应仓库中的哪个范围 + object_format, // 当前为 sha1 + commit_oid, // 固定 commit + root_tree_oid // 固定 tree +} + +NamespaceBinding { + mount_path, // 在 Mega 中出现的位置 + source_snapshot, // 指向哪个仓库的哪个固定 commit/tree + source_subpath, // 从该仓库的哪个子目录开始挂载 + policy // mutable 或 immutable_release +} + +NamespaceView { + schema_version, + instance_id, + native, // 主仓的 SourceSnapshot + bindings_root, // 全部 NamespaceBinding 的索引根 + overrides_root, + materialization_policy +} +~~~ + +V100 对应的数据关系是： + +~~~text +NamespaceView V100 +├── native → SourceSnapshot(main, M10, MT10) +└── bindings_root + └── /third-party/lib + └── NamespaceBinding(lib, L20, LT20, source_subpath=src) +~~~ + +更新到 L21 时，只会创建新的依赖仓 SourceSnapshot、Binding 和 bindings_root，再由它们计算出 V101。M10 对应的数据继续复用。 + +V100、V101 是便于讨论的名字。协议中的真实版本 ID 是 NamespaceView 规范化内容的 SHA-256，例如 sha256:abcd...。只要主仓、任一挂载仓库、挂载路径或政策不同，计算出的版本 ID 就不同。 + +### 4. 一次文件读取 + +当 V100 workspace 读取 /third-party/lib/parser.rs 时： + +~~~text +V100 + → 在 bindings_root 中找到最长匹配 /third-party/lib + → 取出固定的依赖仓快照 L20 / LT20 + → 拼接 source_subpath=src 与剩余路径 parser.rs + → 读取 LT20 中的 src/parser.rs +~~~ + +整个过程不查询依赖仓当前 main，也不查询 latest。因此，即使最新版本已经是 V101，V100 workspace 仍然稳定读取 L20。 + +## 这些文件怎样高效传给 ScorpioFS + +版本确定后，源码小文件按需组成 tar + zstd 小包；ScorpioFS 按文件哈希复用缓存，更新时只请求缺少的文件。大文件单独按块读取。 + +完整示例见 [文件传输设计](scorpiofs-transfer-design.md)：它解释 1,000 个源码文件如何分包、只改 10 个文件时怎样复用，以及 Mega 和 ScorpioFS 各自需要增加什么。 + +## Mega 里的几种目录怎样更新版本 + +Mega 中的路径来源不同，但用户不需要为每种来源学习一种版本。Mega 把它们统一放进同一个 monorepo 版本中： + +| 目录情况 | 例子 | 什么变化会产生新版本 | +| --- | --- | --- | +| 主仓自己的目录 | /project/app | 已合并的主仓内容发生变化 | +| 主仓中可被单独 checkout 的子目录 | /project/team-a | 子目录的变更真正合入对外可见的主仓；仅创建开发候选版本不会推进全库版本 | +| 挂载进来的独立仓库 | /third-party/lib | 该仓库被选为对外可见的分支更新；其他开发分支更新不推进全库版本 | +| 同时包含主仓文件和挂载目录的父目录 | /project | 任一可见子项变化，或者挂载关系变化 | + +这里的关键不是给每种目录单独发一个“全库版本”，而是任何对外可见的变化发生后，Mega 都重新发布一份完整清单。没有变化的部分继续指向原来的 commit，因此不需要复制文件。 + +## 为什么必须由 Mega 发布 + +Mega 是唯一同时知道以下信息的一方： + +- 哪些目录来自主仓； +- 哪些目录来自其他仓库； +- 外部仓库挂载到了什么路径； +- 哪个分支或 commit 当前应当对用户可见； +- 一次 push、merge 或目录调整何时真正成功。 + +ScorpioFS 只看最终目录，无法可靠推断过去某一时刻的挂载关系。如果 Mega 当时没有记录“主仓 M10 应该搭配依赖仓 L20”，客户端事后无法补出这个答案。 + +所以职责划分是： + +| 组件 | 负责什么 | +| --- | --- | +| Mega | 在写入成功时发布整个 monorepo 的版本 | +| ScorpioFS | 固定一个版本并按需读取其中的文件 | +| Libra | 负责开发者侧的 commit、fetch 和 push；不定义整个 monorepo 的版本 | + +## 一个版本里保存什么 + +对外可以把它理解成一张很小的版本清单： + +~~~text +版本 V100 +├── 主仓：M10 +├── /third-party/lib：L20 +└── /toolchains/rust：R7 +~~~ + +清单保存的是已经确定的 commit，而不是“以后读取时再看 main 分支”。这样，同一个版本不会随分支移动而改变。 + +版本号由清单内容决定。以下任一内容变化，都会得到一个新版本： + +- 主仓 commit 变化； +- 某个可见挂载仓库的 commit 变化； +- 挂载路径增加、删除或移动； +- 哪个分支负责提供可见内容的规则变化； +- 目录从普通开发目录变为不可修改的 release 目录。 + +Mega 可以提供 latest，表示最近一次成功发布的版本。latest 会指向新版本，但 V100 这样的具体版本永远不变。生产构建应先把 latest 解析成具体版本，再固定使用该版本完成整个任务。 + +## 什么时候发布新版本 + +任何会改变用户所见目录的操作，都必须经过同一个发布入口。例如： + +- 主仓合并了一个变更； +- 挂载仓库的选定分支收到 push 或网页编辑； +- 新增、删除或移动挂载目录； +- 修改了哪个分支对外可见； +- 修改目录的 release 政策。 + +如果某个仓库的非选定分支发生变化，而它没有影响当前目录，则不需要发布新的 monorepo 版本。 + +## 如何避免只更新一半 + +发布时最危险的情况是：Git 分支已经更新，但 monorepo 版本没有更新；或者版本已经发布，分支更新却失败。两种情况都会让用户看到无法解释的状态。 + +因此，Mega 必须把它们作为一次操作提交： + +~~~text +检查写入基于哪个旧版本 + ↓ +准备新的 Git 对象和完整版本清单 + ↓ +同时更新分支与 monorepo 版本 + ↓ +全部成功才对调用方返回成功 +~~~ + +如果两个写入同时从同一个旧版本开始，最多只能有一个成功。失败的一方需要基于最新版本重新计算，不能悄悄覆盖先完成的写入。 + +如果服务端已经提交成功，但响应在网络中丢失，客户端重试时应拿到第一次操作的结果，而不是再次发布一个版本。 + +## release 目录如何处理 + +release 目录采用已经确认的规则：**首次发布后不可修改**。 + +目录是否为 release 由 Mega 的显式配置决定，不能仅根据目录名猜测。例如，名字包含 1.2.3 的目录不会自动变成 release。 + +- 普通开发目录可以继续更新；每次更新产生新版本，旧版本不变。 +- release 目录首次发布后，push、网页编辑和管理接口都必须拒绝修改。 +- 需要发布新内容时，创建新的 release 目录。 +- 已发布的 release 目录不能通过改回普通目录来绕过限制。 + +## ScorpioFS 如何读取 + +ScorpioFS 启动一个 workspace 时指定具体的 monorepo 版本。之后每次读取都带着这个版本向 Mega 请求文件。 + +Mega 根据该版本保存的清单判断路径属于主仓还是哪个挂载仓库，并从清单指定的 commit 中读取文件。读取过程中不能再查询当前默认分支，也不能用今天的挂载关系解释旧版本。 + +如果旧版本需要的对象已经丢失，Mega 必须明确报错，不能自动回退到 latest。静默回退会让构建成功，却失去可重现性。 + +同一时间可以有多个 workspace 分别使用 V100、V101。新版本发布不会改变已经固定在 V100 上的 workspace。 + +## 默认关闭与历史保留 + +按已确认的安全策略，按版本读取的新接口默认关闭。只有完成以下配置后才能启用： + +- 哪些用户或任务可以读取哪些仓库和路径； +- 历史版本和 Git 对象保留多久； +- 正在使用的 workspace 如何声明“这个版本暂时不能回收”； +- Git 压缩对象依赖的基础对象如何一并保留。 + +知道某个 Git 对象 ID 不等于拥有读取权限。即使对象在存储中存在，Mega 仍需确认它属于请求中的版本，并检查调用方当前是否有权读取。 + +## 使用者最终看到什么 + +理想情况下，普通开发者不需要理解内部的数据表和索引结构，只需要处理两个动作： + +1. Mega 在写入成功后返回新的 monorepo 版本。 +2. ScorpioFS 使用这个具体版本创建或更新 workspace。 + +可以用一句话判断实现是否正确： + +> 给定同一个 monorepo 版本，无论何时、在哪台机器读取，都应得到同一套主仓与挂载仓库文件；拿不到时明确失败，绝不拼出一套“差不多”的目录。 + +## 常见问题 + +### 为什么不能只用主仓 commit + +因为挂载仓库有自己的 commit，它们不包含在主仓 commit 中。主仓 commit 只能固定主仓文件，不能固定整个目录。 + +### 为什么不让 ScorpioFS 自己记录版本 + +ScorpioFS 不负责 Mega 的写入，也不知道一次主仓、挂载仓库或挂载规则更新何时原子完成。由客户端分别记录容易再次产生新旧混合状态。 + +### latest 可以直接用于整个构建吗 + +不应在每次读取时重新查询 latest。构建开始时可以查询一次，然后把得到的具体版本固定到 workspace；否则 latest 在构建中途移动，问题会重现。 + +### 发布新版本会复制整个 monorepo 吗 + +不会。版本清单只记录 Git commit 和挂载关系，文件内容仍由 Git 对象存储复用。实现需要高效处理大型挂载表，但不会为每个版本复制一份完整目录。 + +### 旧版本可以被修改吗 + +不可以。新写入只会产生新版本。latest 可以前进，具体版本不会被覆盖。 + +## 实施状态 + +目前已经完成并有测试覆盖的基础能力包括： + +- 按固定仓库版本读取，而不是读取当前分支； +- 识别仓库和目录归属，防止跨仓库误读对象； +- Mega 与 ScorpioFS 共用同一种版本清单格式； +- 大型挂载表的持久化与局部更新； +- 并发写入检查、原子更新、请求重试恢复和可靠通知的事务核心。 + +以下部分尚未接入完整生产链路，因此 PR 继续保持 Draft： + +- 真正组合主仓与所有挂载仓库的发布服务； +- 所有 push、merge、网页编辑和管理入口的统一接入； +- release 不可修改规则在所有写入口上的执行； +- 历史对象租约、回收和权限控制； +- 对外 HTTP API； +- Mega 与真实 ScorpioFS/FUSE 的双版本端到端验证。 + +## 实现细节在哪里 + +上文只解释产品行为。需要实现或评审底层协议时，再阅读以下文档： + +- [服务端实施细节](spec/namespace-snapshot-spec.md) +- [版本清单编码](spec/namespace-manifest-v1.md) +- [挂载索引](spec/namespace-index-v1.md) +- [发布事务核心](spec/namespace-publication-core.md) + +这些文档中的 source ID、scope proof、radix index、receipt、outbox 等术语都是实现手段，不是要求普通使用者学习的新概念。 diff --git a/docs/scorpiofs-transfer-design.md b/docs/scorpiofs-transfer-design.md new file mode 100644 index 000000000..b5f514d0e --- /dev/null +++ b/docs/scorpiofs-transfer-design.md @@ -0,0 +1,139 @@ +# Mega 如何把文件高效传给 ScorpioFS + +状态：设计提案，尚未实现。本文接续 [monorepo 版本设计](monorepo-versioning-design.md)。工作负载已确认以大量源码小文件为主；大文件允许信任 Mega 校验整文件后生成的分块表。 + +建议采用 HTTPS/HTTP/2，目录信息用 JSON，小文件用 **tar + zstd 小包**，大文件按块读取。所有请求固定同一个 monorepo 版本。 + +面向论文，包格式是工程选择；研究重点是根据版本变化、当前缓存和需求选择传输方式，并证明这种选择在何种条件下有效。相关工作、正确性论证和实验方案见 [研究设计](spec/scorpiofs-transfer-research.md)。包大小和静态阈值目前用于建立基线，不是已证实最优参数。 + +## 从一次构建看传输过程 + +假设 V100 包含以下内容： + +~~~text +/project/app/src/ 1,000 个源码文件，每个 8 KiB +/third-party/lib/ 挂载的依赖仓 +/toolchains/compiler.bin 一个 3 GiB 文件 +~~~ + +ScorpioFS 挂载 V100 时只获取版本和当前需要的目录，不扫描整个 monorepo。 + +构建访问 src 时，Mega 按页返回文件名、类型、准确大小和 Git blob ID。假设构建明确需要这 1,000 个文件，ScorpioFS 接着检查本地缓存： + +| 情况 | 向 Mega 请求什么 | +| --- | --- | +| 这些文件一个都没缓存 | 选择目录对应的小包；每包最多 128 个文件 | +| 已缓存 990 个，只缺 10 个 | 把这 10 个文件列出来，请 Mega 组成一个包 | +| 只打开其中一个文件 | 立即读取这个文件；不为一次读取强制下载整个目录 | +| 多个编译进程同时需要同一文件 | 合并为一次下载，完成后一起使用 | +| 读取 compiler.bin 中的一小段 | 下载覆盖这一段的文件块 | + +这 1,000 个源码共 7.8125 MiB。全部缺失时，按每包 128 个文件计，需要 8 个包，而不是 1,000 次独立文件下载。目录分页请求另计，不能把它们藏进这个数字。这里比较的是请求数量，不是假设串行下载，也不是实测加速比。 + +这个例子假设文件需求已由构建输入清单给出或足够并发地出现。纯按需 FUSE 可能串行发现文件，不能保证自然凑出 8 个包；实验必须分别测量这两种情况。 + +## “部分打包”具体指什么 + +Mega 不会为整个 monorepo 生成一个大压缩包。包按当前需要的局部文件生成，默认在同一仓库、同一读取范围内： + +~~~text +V100 +├── app/src 的小包 A：128 个文件 +├── app/src 的小包 B：128 个文件 +├── ... +└── lib 的小包：单独组织 +~~~ + +初始建议：单个小文件不超过 256 KiB；一个包通常接近 1 MiB，最多 128 个文件、4 MiB 文件原始内容。大目录自然拆成多个包。数字是待基准测试调整的默认值。 + +Mega 可以缓存热点目录的小包。多人构建同一目录时直接复用，避免每次重新压缩。没有现成包时，按客户端提交的缺失文件列表构建。只为被请求的目录生成，后台预热也受预算限制。 + +目录返回的预打包提示只是候选。ScorpioFS 先比较“包里还有多少没缓存”和“直接请求缺失文件的成本”，不会为了一个文件下载一个几 MiB 的包。 + +## 包里是什么 + +复用标准 tar 格式，把一批 Git 对象放在一起，再用 zstd 压缩整个小包： + +~~~text +package.tar.zst + 解压后： + ├── manifest.json + ├── objects/blob/ + ├── objects/blob/ + └── objects/blob/ +~~~ + +文件内容直接使用二进制字节，不做 Base64。共同压缩让相似源码有机会提高压缩率，收益需要测量。每个包独立解压，不依赖另一个包或旧版本文件。 + +manifest.json 的教学示例： + +~~~json +{ + "schema_version": 1, + "object_format": "sha1", + "objects": [ + { "oid": "A", "kind": "blob", "raw_size": 8192 }, + { "oid": "B", "kind": "blob", "raw_size": 8192 } + ] +} +~~~ + +A、B 是简写，协议实际使用完整 Git OID。清单不记录会过期的权限、latest 或 workspace 信息，因此同一批对象组成的包可以跨版本复用。 + +ScorpioFS 逐个读取包成员，核对清单和大小，再计算 Git blob 哈希。验证完成的对象进入本地内容缓存。它通过对象 ID 存储文件，不把 tar 直接解包到用户目录。 + +目录路径、可执行位和符号链接语义仍来自固定版本的目录元数据；包只运输对象内容。 + +## 更新到 V101 时怎样省流量 + +假设只修改了 src 中的 10 个文件，其他 990 个文件内容没有变： + +~~~text +V100 的本地缓存：A1、A2、...、A1000 +V101 的目录信息：其中 10 个 blob ID 变化 + +ScorpioFS： + 复用 990 个已有 blob + 把 10 个新 blob 组成一次请求 + Mega 返回一个 tar.zst 小包 +~~~ + +新版本的小包布局即使发生变化，也不要求客户端重下整个目录。缓存复用的基本单位是文件对象，包只是运送这些对象的容器。10 个 8 KiB 文件的原始内容是 80 KiB，实际线上字节还包括包头并受压缩率影响。 + +同一个字节内容在同一授权域内可复用；新版本的路径映射、目录分页和权限必须重新确认。 + +## 大文件怎样处理 + +3 GiB 文件不会进入源码小包。Mega 先校验它的 Git 整文件哈希，再保存一个分块表： + +~~~json +{ + "blob_oid": "COMPILER", + "raw_size": 3221225472, + "chunk_size": 1048576, + "chunk_count": 3072 +} +~~~ + +分块表分页列出每块的序号、长度和 SHA-256。ScorpioFS 只取覆盖 read(offset, length) 的块。例如，从 100 MiB 位置读取 64 KiB，只需要第 100 块的 1 MiB 内容；恰好跨块时会需要两块。 + +这是带取舍的选择：块较大，顺序读取请求较少；只读几 KiB 时会多下载一些。默认 1 MiB，之后用真实工具链访问记录调整。 + +客户端通过 HTTPS 信任 Mega 提供的“Git blob 与分块表的对应关系”，再独立核对每块哈希。它不会声称一个分块哈希就能证明整个 Git blob。这个信任方式已获确认。 + +如果对象在 Git pack 中采用差量存储，Mega 必须先还原并建立可随机访问的数据。不能每读一个块都重新解压整个 3 GiB 文件。这部分成本在对象接收或受限后台任务中处理。 + +## 两端各自增加什么 + +| Mega | ScorpioFS | +| --- | --- | +| 固定版本的目录分页，一次给出文件大小和 OID | 将目录条目交给 lookup/getattr/readdir，不为 stat 下载正文 | +| 缺失文件批量打包接口 | 汇总同时发生的缓存缺失，优先处理当前阻塞的读取 | +| 热点目录的小包缓存 | 选择现成包或精确缺失列表，限制多余预取 | +| 大文件分块表及可随机读取的存储 | 按偏移读取、校验和缓存文件块 | +| 每次传输检查版本、权限及租约 | 按授权域隔离内容缓存，按版本隔离路径缓存 | +| 限制打包 CPU、磁盘、数据库并发 | 限制网络并发、内存、磁盘和预取；断线只重试未完成对象 | + +HTTP/2 提供连接复用和流量控制，但不会自动替我们决定预取什么或限制打包 CPU；两端仍需调度。[HTTP/2 标准](https://www.rfc-editor.org/rfc/rfc9113.html) + +首版先做目录元数据和按需小包，再做热点包复用及大文件分块。这样可以先验证源码构建的主要收益。完整字段、接口、校验规则和测试计划见 [传输协议 Spec](spec/scorpiofs-transfer-v1.md)。 diff --git a/docs/spec/namespace-index-v1.md b/docs/spec/namespace-index-v1.md new file mode 100644 index 000000000..a310a663d --- /dev/null +++ b/docs/spec/namespace-index-v1.md @@ -0,0 +1,94 @@ +# Namespace binding index v1 + +Status: implemented internal index foundation, 2026-09-06. This document fixes the +radix codec used by `ceres::application::snapshot::radix`; it does not enable a +namespace capability. Binding values, complete view manifests, publication +transactions, authenticated public cursors and retention are separate layers. + +## Key and node encoding + +Keys are canonical `RepoPath` values (UTF-8, at most 4096 bytes, each component at +most 255 bytes). `/` encodes as zero bytes. For any other path, remove the leading +slash, replace each remaining slash with NUL, and append NUL. Thus `/a/b` becomes +`61 00 62 00`; `/rust` is not an ancestor of `/rust_v1`. No Unicode normalization +or case folding occurs. Ordering is unsigned lexicographic encoded-byte order, +not locale or slash-separated string order. Ancestors precede descendants. + +Each node is a compressed label, optional immutable binding digest, and sorted +byte-edge children. Canonical bytes, in order: + +| Field | Encoding | +| --- | --- | +| domain | ASCII `mega.namespace-radix.v1` followed by NUL | +| label length + label | u16 big-endian byte length, then that many bytes | +| binding present | u8, exactly 0 or 1 | +| binding digest | 32 raw SHA-256 bytes, only if present | +| child count | u16 big-endian, at most 256 | +| each child | u8 edge followed by 32 raw SHA-256 digest bytes; strictly increasing edges | + +A child label includes its incoming edge as its first byte. Labels can split a +UTF-8 sequence; only complete value keys are decoded as repository paths. Each +label and assembled key is bounded by 4096 bytes; the complete node by 16384 +bytes. A valueless node with one child is compressed into that child. A valueless +node with no children must have an empty label. Trailing bytes, malformed tags, +unsorted or duplicate children, invalid value paths and wrong incoming labels +are errors. Digests are `sha256:` plus lowercase hex of SHA-256 over the entire +canonical byte sequence, including the domain. Unknown schemas fail closed. + +The empty root is an implicit canonical node with empty label, no value and no +children; it requires no stored row: + +`sha256:18946486089198dfa8eeb70fa90e04b137c579dc08ae1e6f8bceafc0d35ef677` + +Independent .NET SHA-256/framing vectors are committed in +`ceres/tests/fixtures/snapshot/namespace-radix-v1.json`. They include empty, leaf +and branch nodes, with exact bytes and digests. Rust tests decode/re-encode and +hash all vectors. The ScorpioFS client does not yet decode this index. + +## Operations and persistence + +`update(root, path, value)` copies changed ancestors only; deletion compresses +the result. A no-op writes no nodes. Insertion order and delete/reinsert produce +the same root for the same mapping. Old roots and their nodes are never mutated. +`get` and `longest_prefix` use component-aware routing without registry reads. +The implementation uses iterative traversal, not path-depth recursion. + +`page(root, prefix, after, limit)` returns up to 256 bindings in encoded-key +order, plus `has_more`, pruning nonintersecting subtrees. `after` must be inside +the prefix and is exclusive. This is an **internal keyset primitive**, not an +authenticated public cursor. HTTP cursors must additionally bind view, prefix, +query, schema and expiry; passing an arbitrary raw `after` externally is not +that contract. Each call uses the explicit immutable root throughout. + +The `NodeStore` boundary verifies size and digest before decoding. The Jupiter +adapter inserts immutable rows using the caller's database transaction. SQL +enforces the payload size; the facade additionally verifies schema, digest and +conflicting existing bytes. An absent or corrupt node is an availability error, +never an empty directory. Metadata retention must eventually trace index child +edges and binding/view references; insert-only storage is not a GC policy. + +## Reproducible checks and limits of the evidence + +Local WSL debug runs (Rust stable, 2026-09-06) use an independent ordered-map +oracle for mutation traces and fixed structured trees for scale fixtures: + +| Bindings | Single update reads/writes | Bytes read/written | Largest accessed node | Page 32 reads | One prefix reads | +| --- | --- | --- | --- | --- | --- | +| 10,000 | 5 / 5 | 1515 / 1515 | 372 B | 42 | 5 | +| 1,000,000 | 7 / 7 | 2235 / 2235 | 372 B | 44 | 7 | + +These are logical `NodeStore` calls, not SQL statement counts or HTTP latency. +The million-entry test constructs an in-memory six-digit decimal fixture; it is +not a million-row database publication benchmark. Isolated process peak was +about 356 MiB, including the entire in-memory fixture; the combined concurrent +snapshot suite reached about 413 MiB. Neither is per-request service memory. +Worst-case path depth/fanout differs from this decimal fixture and remains +bounded by the codec/key limits, not by these measured averages. + +Run `cargo test -p ceres --lib snapshot::radix::tests::million --locked -- +--ignored --nocapture` for the scale gate. SQLite and PostgreSQL tests also +exercise real migrated node tables, transaction rollback, reconnection, stable +source identity, scope proofs and old/new root reads. They do not prove atomic +ref/view publication, public cursor isolation, authorization, leases or FUSE +behavior. MG12 and MG15 are therefore only partially covered; do not mark the +broader namespace acceptance suite complete. diff --git a/docs/spec/namespace-manifest-v1.md b/docs/spec/namespace-manifest-v1.md new file mode 100644 index 000000000..9aaea4af1 --- /dev/null +++ b/docs/spec/namespace-manifest-v1.md @@ -0,0 +1,81 @@ +# Namespace manifest identity v1 + +Status: shared codec implemented and tested, 2026-09-06. This is a content +identity contract, not a claim of publication, authorization, leases or FUSE +integration. Both repositories consume the same `namespace-v1.json` fixture; +the committed PowerShell 7 generator independently frames bytes and hashes with +.NET, without calling the Rust implementation. + +## Binding + +JSON has exactly `mount_path`, `source_snapshot`, `source_subpath` and `policy`. +The first is a canonical RepoPath; source_subpath is a canonical RelativePath +relative to the attested source scope. Their source-side composition must still +fit the 4096-byte absolute-path limit. Tree existence, source membership, +ancestor/descendant binding conflicts and release enforcement are publisher +checks, not proofs supplied by this structural codec. + +Canonical bytes are ASCII `mega.namespace-binding.v1` plus NUL, followed by: + +1. Mount path: u32 big-endian byte length, then UTF-8 bytes. +2. Full canonical SourceSnapshot bytes from source-snapshot-v1, framed by u32 + big-endian byte length. This is not JSON and not merely the source UUID. +3. Source subpath: u32 big-endian byte length, then UTF-8 bytes. +4. Policy u8: 1 = `mutable`, 2 = `immutable_release`. + +Policy is explicit and part of identity; it is never guessed from a numeric +directory name. **D2 is confirmed:** an explicitly marked release directory +cannot change content after its first publication; ordinary development +bindings may evolve. A codec that can encode both values does not itself enforce +this rule on writers. + +## View + +JSON has exactly `schema_version`, `instance_id`, `native`, `bindings_root`, +`overrides_root` and `materialization_policy`. Schema version must be integer 1. +Instance ID is a distinct non-nil canonical UUID type, not a source ID. +The native SourceSnapshot must have root scope `/`; the server must separately +attest that its source is the instance's native backend. + +Canonical bytes are ASCII `mega.namespace-view.v1` plus NUL, followed by: + +1. Schema version: u16 big-endian, exactly 1. +2. Instance UUID: u32 big-endian byte length, then canonical lowercase UUID text. +3. Full canonical native SourceSnapshot bytes, framed by u32 big-endian length. +4. Bindings root: 32 raw SHA-256 bytes, with no textual prefix. +5. Overrides presence: u8 0 for absent, or u8 1 followed by 32 raw digest bytes. +6. Materialization policy u8: 1 = `git_raw_v1`. + +`git_raw_v1` identifies raw Git projection without implicit LFS hydration or +submodule expansion. It is not permission to traverse arbitrary external +symlinks. Overrides are representable in the codec; the reader must explicitly +reject that capability until the override route semantics are implemented. +The absent root is distinct from a present empty-index root. + +No timestamp, actor, operation ID, publication sequence, parent view, floating +ref, lease or client generation is hashed into a view. A different commit with +the same tree changes provenance and therefore changes view_id. Re-publishing +identical content can reuse view_id while publication metadata remains separate. + +## Strict decoding and cross-repository evidence + +Every complete manifest is limited to 16384 bytes. Hash identity is `sha256:` +plus lowercase hex SHA-256 of the entire domain-separated canonical byte +sequence. Binary decoding rejects truncation, length overflow, trailing bytes, +unknown schema/policy/optional tags, invalid UTF-8 or paths and mismatched +domains. JSON rejects unknown fields and passes through the same structural +validation as constructors. Decoding bytes is not digest verification against a +requested ID; the content-store/read boundary must do that separately. + +Golden view without overrides: +`sha256:3c8632afb308bf562973b3af517ae5d0a27c05651f3f7511f91e16d7ad8f1231`. + +Golden mutable binding: +`sha256:adebe124b05761074c9460ed20426acf3023645e2bfa7e46b12239da68b14a88`. + +Both repositories pass five codec tests: independent binary/JSON vectors, +identity changes with provenance/routing/instance/policy, JSON rejection, +all-prefix truncation and malformed tags/lengths, and maximum-length paths. +Mega uses SHA-2 and ScorpioFS uses ring, while the oracle uses .NET SHA-256. +This closes the shared manifest-identity subtask, not the full G01–G06/V01–V18 +acceptance suite. diff --git a/docs/spec/namespace-publication-core.md b/docs/spec/namespace-publication-core.md new file mode 100644 index 000000000..3aa3d72f8 --- /dev/null +++ b/docs/spec/namespace-publication-core.md @@ -0,0 +1,103 @@ +# Namespace publication transaction core + +Status: implemented storage core, 2026-09-06; **not an enabled publisher API**. +The application composer, all production writer integrations, release-policy +enforcement, prepare/retention pins and authorization remain required before +the namespace capability can be announced. + +## Ownership and ordering + +`PublicationStorage::begin(request, expected_head, writer_epoch)` owns one +database transaction. Its first write reserves the unique +`(actor_domain, operation_id)` row. A duplicate committed request returns its +receipt before exposing any ref-writing handle. Reusing the same committed key +with a different request digest or instance is a conflict. Failed/aborted +transactions leave no operation reservation or success receipt. + +The request digest is supplied by a trusted application adapter and MUST cover +the complete canonical mutation plan: fixed base/head, expected refs, binding +policy/read set and prepared content identities. The storage facade cannot +infer these fields from an opaque digest. The authenticated actor domain must +not be accepted from untrusted request JSON. Receipt reads require current +authorization independently of the operation key. + +A ready result owns `PublicationTransaction`. Writers can borrow its underlying +transaction for conditional refs, prepared metadata, scope attestations and +index nodes, but cannot obtain ownership and independently commit it. Explicit +abort or dropping the owner rolls back. Publication's `finish` is the only +commit path exposed by this wrapper. + +`finish` validates the prepared view identity/byte bound and same instance, then +stages insert-only view bytes, conditional head update, publication history, +operation result and outbox event. They commit together with the borrowed +transaction's ref changes. A database error after head CAS still rolls back +the head, view and refs. No notification is dispatched before COMMIT. + +## Compare-and-swap and receipts + +The head condition includes instance, expected sequence, expected view ID and +writer epoch. Bootstrap is an insert-if-absent head, not an upsert. Sequences and +epochs are positive SQL BIGINT values and sequence increment checks overflow. + +When the descriptor is unchanged, the operation may be a no-op for namespace +publication: preserve sequence/view and do not insert publication/outbox rows. +It STILL executes the head/epoch fence. For example, a non-selected branch may +change without changing the default namespace view. Determining that the view +really represents the complete post-write state belongs to the application; +the storage facade must not be used to hide a selected-ref mutation. + +`GitDbStorage::update_ref_if_unchanged` adds one conditional SQL update on +repo ID, fully qualified ref name and expected object ID, returning whether +exactly one row changed. It accepts the publication transaction and does not +silently rebase/retry. Existing legacy writers are not yet switched to this +method. The caller must abort the whole publication if any required ref/read +condition fails. + +A COMMIT error is reported as an uncertain outcome, not a proven rollback. +Look up the original actor/operation/request digest on a new connection before +retrying. Receipt replay never dispatches a second ref mutation or outbox +event. Outbox rows have unique event IDs and pending/delivered state, but the +delivery worker and external side effects are not implemented by this core. + +A writer_epoch column does not fence an old binary that never checks it. +Maintenance cutover and an audit of every production writer remain G04/G05 +requirements; a passing storage test cannot establish those conditions. + +## Schema and reproduction + +The additive migration `m20260906_160000_namespace_publication` creates +namespace_view, namespace_head, namespace_publication, snapshot_operation and +namespace_outbox. It creates no initial head/catalog and enables no feature. +Generated Callisto fields were produced with sea-orm-cli 2.0.2 from the actual +SQLite migration schema; PostgreSQL tests verify the same runtime schema. + +View payloads are bounded to 16 KiB in SQL and checked against their SHA-256 ID. +The application supplies the already validated namespace-manifest-v1 codec. +An opaque-byte storage fixture is not proof that the manifest describes the +actual native/import objects. No foreign-key cascade from a mutable ref or +registry path deletes published metadata. Retention/GC and referential audits +must be supplied by the full publisher before deployment. + +Use the explicit loopback disposable PostgreSQL URL described in +[jupiter-migrate](../../jupiter-migrate/README.md), then run: + +```bash +cargo test -p jupiter --lib publication_storage --locked -- --include-ignored --nocapture +cargo test -p jupiter-migrate --lib snapshot --locked -- --include-ignored --nocapture +``` + +The six publication tests cover SQLite lifecycle, duplicate-key concurrency and +expected-old competition, plus PostgreSQL lifecycle/reconnect, concurrent +duplicates/expected-old writers and an independent-connection epoch change. +Shared lifecycle checks also inject failure after head CAS, drop an uncommitted +transaction, reject different-payload replay, preserve old views and verify +no-op ref writes. They use the REAL import_refs and new publication tables. +PostgreSQL tests create fresh random schemas, retain diagnostics and never +refresh a supplied database. Tests do not cover a process/host power loss, +external payload durability, notification delivery, source/path permissions or +release-policy bypass through actual production routes. + +The CI focused snapshot job runs these PostgreSQL tests explicitly instead of +silently skipping ignored tests. MG06/MG09/MG15 have additional storage-level +evidence; the broader acceptance IDs remain incomplete until application and +real-service integration are tested. diff --git a/docs/spec/namespace-snapshot-spec.md b/docs/spec/namespace-snapshot-spec.md new file mode 100644 index 000000000..d558f78d2 --- /dev/null +++ b/docs/spec/namespace-snapshot-spec.md @@ -0,0 +1,285 @@ +# Mega Namespace Snapshot：服务端实施 Spec + +> 第一次阅读请从 [Mega 的 monorepo 版本发布设计](../monorepo-versioning-design.md) 开始；本文保留实现级契约和验收细节。 + +状态：Draft v0.4，2026-09-06。基线 `c4c79bc195541a13ac1505b94728c81a8ff3d603`。本文是目标设计，不是当前服务端能力清单。已实现的基础包括 import 固定 commit 解析、source identity/scope 证明、持久化 bounded radix 索引以及两仓共享的 [namespace manifest 编码](namespace-manifest-v1.md)；未部署或开放 snapshot capability。D1（完整 native + import 原子组合视图）、D2（显式 release 目录发布后不可变）与 D4（安全启用门槛）已获用户确认；D3 尚待确认。 + +跨仓跟踪：[ScorpioFS #55](https://github.com/gitmono-dev/scorpiofs/issues/55)，关联 [#42 Snapshot](https://github.com/gitmono-dev/scorpiofs/issues/42)。配套客户端规范为 ScorpioFS 仓库的 `docs/spec/monorepo-versioning.md`；本文细化 Mega 的写入、存储、API 与迁移责任。基础实现持续审阅入口为 [Mega Draft PR #2181](https://github.com/gitmono-dev/mega/pull/2181) 与 [ScorpioFS Draft PR #56](https://github.com/gitmono-dev/scorpiofs/pull/56)；基础契约及测试入口见 [source-snapshot-v1.md](source-snapshot-v1.md)。完整 namespace 发布事务、所有写入者接入、GC/lease、HTTP/FUSE 联调及受控更新仍是未完成的交付门槛，不能由基础单测或 Draft PR 创建替代。 + +## 1. 交付目标与非目标 + +Mega 提供两个可独立验收的能力： + +- `source-snapshot.v1`：一个经过验证的 source/scope/commit，其 tree/blob 读取不再解析实时分支。 +- `namespace-snapshot.v1`：服务器发布原生 root 与 import 挂接索引组成的不可变视图，`latest` 只读发布指针。 + +先修单 source 历史读取，但它不是全库快照完成的证据。整个 view 必须同时固定内容和路由；ScorpioFS 的只读挂载不能补造 Mega 从未记录的挂接历史。 + +非目标：Git 全局线性历史重写、任意来源的原子跨仓 commit、自动展开 LFS/submodule、运行进程透明换代、迁移旧历史时猜测缺失的依赖版本。Libra 管 VCS，Mega 管发布，ScorpioFS 管投影；不让 Mega 接管工作区 upper/HEAD/index。 + +## 2. 源码核对结果 + +下列链接相对本仓库，行号以基线为准。实现时重新核对，不以注释代替执行路径。 + +| 证据 | 当前行为 | 改动结论 | +| --- | --- | --- | +| [ImportApiService](../../ceres/src/application/api_service/import_api_service.rs)，`get_root_tree`，约 117 行 | refs 参数被忽略，读取当前默认分支 | 新 resolver 必须显式解析 commit/ref；保留 legacy latest 行为于旧 API | +| [tree_ops](../../ceres/src/application/api_service/tree_ops.rs)，`get_binary_tree_by_path` | 先查当前 path，再校验可选 oid | 新接口从固定 root/OID 读；不能把旧接口的 oid 校验包装成历史读取 | +| [api_handler](../../mono/src/api/mod.rs)，约 82 行 | 按当前 import_dir 与 `git_repo` 切换 handler | snapshot router 不调用这个实时路由函数决定历史路径归属 | +| [mega_commit](../../jupiter/callisto/src/mega_commit.rs) 与 [mega_refs](../../jupiter/callisto/src/mega_refs.rs) | commit 没有 scope 字段，ref 有 path；scope clone 可生成新 commit | 增加持久化 scope 证明，不能靠最新 ref 反推所有历史 commit 的 scope | +| [共享事务](../../jupiter/src/storage/mod.rs)，`begin_db_transaction`，约 327 行 | monorepo/import 元数据使用同一应用连接开事务 | 新 namespace 元数据加入此事务；当前不必引入跨数据库提交协议 | +| [import post-receive](../../ceres/src/application/code_edit/post_receive/import.rs) | 分支 ref 和原生占位路径 attach 有联合事务、root CAS | 保留现有保护，并增加 binding/published pointer/逐 ref expected-old 校验 | +| [git_db_storage](../../jupiter/src/storage/git_db_storage.rs)，`update_ref_in_txn` | 读行后写新值，接口没有 expected-old 参数 | root CAS 不能替代每个 import ref 的并发租约校验 | +| [网页编辑](../../ceres/src/application/api_service/import_api_service.rs)，`save_file_edit`，约 374 行 | 先生成 tree，再读默认 ref，最后单独 update_ref | 一次固定 base 构建修改，使用共同 publisher；并发变化返回冲突，不拼接不同 base | +| [transport 建仓](../../ceres/src/transport/protocol/mod.rs)，约 168 行 | receive-pack 准备阶段可能先保存 git_repo，再接收对象；当前已拒绝删除默认分支 | 登记与已发布 binding 分开；保留默认分支删除保护并在事务内重验 | +| [smart receive-pack](../../ceres/src/transport/protocol/smart.rs)，约 334–403 行 | finalize 成功之后才构造各 command 的 report-status；tag 提前写入 | 不声称现有分支在 finalize 前回报成功；新 publisher 必须仍在成功边界之内 | +| [Mono merge](../../ceres/src/application/api_service/mono/cl/merge.rs)，`apply_update_result` | 事务写 tree/commit/ref，候选 ref 更新在事务前计算 | 接入 publication CAS 和读集校验，避免陈旧候选覆盖并发发布 | + +搜索命中的 `build_trigger/service.rs::create_repo_and_save_ref` 位于 `#[cfg(test)]`，不是新生产建仓入口。当前发现的 artifact GC 管理另一类对象；本次没有验证到覆盖 Git snapshot 闭包的保留协议，不能沿用 artifact GC 的完成状态宣称 Git 对象已被保护。 + +## 3. 身份与 scope 验证 + +### 3.1 不可变身份 + +`SourceSnapshot = {source_id, scope_path, commit_oid, root_tree_oid, object_format}`。 + +单 source 身份的已实现编码、校验器与两仓共享向量见 [source-snapshot-v1.md](source-snapshot-v1.md)。它不替代后续 namespace view/index 编码、发布或保留闸门。 + +- `source_id` 是实例内永久身份，映射 backend kind 与现有 repo_id；删除/改路径不能重用该身份。建议持久 UUID，现有整数 repo_id 只作内部关联。 +- 原生主仓只有一个 source，但可有多个 scope；import 各自独立 source。路径既不是 source ID，也不能供客户端指定任意后端 URL。 +- 直接 root/scope commit 的 `root_tree_oid` 等于验证后的 commit.tree；从一个已证明的 native source 派生子目录时，保留 base commit provenance，root_tree_oid 则是沿固定 base tree 到目标 scope 验证得到的 subtree。这两种证明必须区分，不能无证明替换根树。scope `/project/a` 的 root 已在 a 内，读取 `src/lib.rs` 不能再拼接 `project/a`。 +- 同一 commit 字节可出现在多个有效 scope；证明表是多对多关系，不设 `commit_oid → 唯一 scope` 假设。 +- M1 对象格式只宣布已实现的 SHA-1；类型预留其他算法，未知算法显式拒绝。 + +`NamespaceView = {schema_version, instance_id, native, bindings_root, overrides_root?, materialization_policy}`；`view_id` 是规范化 descriptor 的 SHA-256 digest。binding 内容固定 `{mount_path, source_snapshot, source_subpath, policy}`，不得保存一个待读时解析的 branch 作为内容身份。 + +`publication_seq` 是实例发布序号；`view_id` 是内容/来源描述身份；客户端 `generation/delta_seq` 不由 Mega 分配。发布时间、租约、actor、浮动 selector 放在发布记录，不进入 view hash。未发布的候选 view 也可以有 view_id，但不能冒充 publication_seq。 + +`projection_key` 是客户端的有效投影身份；服务端不以它替代 provenance。客户端还需考虑访问域、合成 stat 策略与 inode 规则。 + +### 3.2 Scope 证明 + +已添加 `source_commit_scope`，逻辑唯一键 `(source_id, scope_path, algorithm, commit_oid)`，保存 root_tree_oid、证明类型和可审计来源（产生该对象的 ref mutation/父 scope 映射/已发布 root）。实际数据库索引使用 scope 的 SHA-256 key 并核对完整路径，避免 PostgreSQL 长路径 btree 限制；所有生产创建/merge 入口的证明写入仍需接入。 + +证明在原生 root/子 scope commit 创建、scope clone 派生、CL 接收、merge 生成路径 commits 时一起记录。已知 root commit 可沿已验证 root 历史建立 root-scope 关系；不能把任意存在于 `mega_commit` 的对象默认视为 `/`。 + +存量历史子 scope 没有可靠证明时返回 `SOURCE_SCOPE_UNVERIFIED`。允许显式管理回填经过验证的映射；不从如今同名目录或已被清理的 child ref 猜测。已固定 descriptor 不因 child ref 被 `remove_none_cl_refs` 清理而失效。 + +### 3.3 Selector + +请求为带类型联合：`published_view(view_id|latest)`、`source_commit(source_id, scope_path, commit_oid)`、`source_ref(source_id, scope_path, full_ref_name)`。ref 必须完整限定 `refs/heads/...` 或 `refs/tags/...`；branch/tag 同名不自动猜测。 + +tag 解析返回 ref OID、必要的 annotated-tag peeling 链与最终 commit。树/blob tag 不是合法 commit selector；循环、超深链或 target 不可用明确失败。原有 tag 表与 ref 表存在两种表示，resolver 需要回归两类创建入口，不能假设所有 ref OID 都直接是 commit。 + +裸 tree 读取是对象操作，不形成可宣称 commit provenance 的 SourceSnapshot。source_ref 只解析一次，返回固定身份后所有后续读取不再跟随 ref。 + +## 4. 挂接索引与目录语义 + +索引输入是已发布 binding，不是每次读 `git_repo + import_refs`。原生树在 import 边界的占位内容由 binding 替换；聚合目录合并原生子项与固定 binding 子项，同名非声明替换冲突拒绝。跨 scope override 不允许穿过 import source 边界。 + +路径以组件匹配；`/rust` 不匹配 `/rust_v1`。v1 采用有效 UTF-8 路径组件，不做大小写折叠/Unicode 归一化；拒绝 NUL、`.`、`..`、重复分隔等非规范输入，非 UTF-8 名称明确返回 unsupported，不静默改名。Git 路径语义不套用 Windows 路径规则。路径编码规则进入 schema 版本。 + +已实现的 [索引基础](namespace-index-v1.md) 是持久化压缩 byte-radix/Merkle trie：组件间使用禁止出现在名称中的 NUL 作为内部边界，内部节点最多 256 个分支，value 保存独立 binding digest。节点大小、最长路径有硬上限，遍历不依赖递归；压缩长 label 仍受节点上限约束。公开分页 cursor、组合策略和保留遍历尚未由这层实现。 + +仅说“重写祖先节点”还不够：根节点若内嵌百万 children，单次更新仍是 O(R)。首个索引 PR 必须证明节点 fanout/大小受限，更新 b 个 binding 的成本受变动 key 长度和受限节点数控制；持久化旧节点继续被旧 view 引用。 + +按 prefix seek 和分页，不在每次 mount 或每页 readdir 扫描全 registry。cursor 绑定 view_id、prefix、最后排序 key、schema 与查询参数并防篡改；续页重新鉴权。ScorpioFS directory handle 绑定该 view，不能跨 view 使用 cookie。 + +百万 binding 测试记录 node reads/writes、bytes、峰值内存和分页工作量，不只报告平均耗时。初始全量建索引允许 O(R)，在线单点发布和小工作集 mount 不允许。 + +## 5. 元数据模型（目标模型，部分已落地） + +当前 additive schema 已包含 source/scope、namespace_node、view/head/publication、operation 与 outbox；binding_head、pin 及完整业务发布策略尚未落地。表存在不代表已部署或启用完整 namespace capability。 + +| 表/存储 | 关键字段与约束 | 用途 | +| --- | --- | --- | +| `snapshot_source` | source_id PK，instance_id/kind/repo_id，状态；repo 身份不可重用 | 与可移动路径解耦；删除后保留 tombstone | +| `source_commit_scope` | §3.2 复合唯一键、root tree、proof | 验证 native scope 和历史读取 | +| `namespace_binding_head` | source_id、当前 mount_path、selected_ref、policy、revision、active/staged；活动路径唯一 | 当前发布政策/配置；不是历史读源 | +| `namespace_node` | digest PK、schema、canonical bytes；不可原地覆盖 | binding trie 与不可变 binding values | +| `namespace_view` | view_id PK、canonical descriptor、native root、bindings_root | 可重放 view；记录可与多次发布关联 | +| `namespace_head` | instance_id PK、publication_seq、view_id、writer_epoch | 唯一默认发布指针和 CAS/fencing 条件 | +| `namespace_publication` | `(instance_id,seq)` 唯一、view_id、parent_seq/view、reason、operation_id | 发布历史，不把 parent 写入 view hash | +| `snapshot_operation` | `(actor_domain,operation_id)` 唯一、request_digest、receipt | 响应丢失时可查询；同 key 异 payload 拒绝 | +| `snapshot_pin` | pin_id、target kind/id、owner、expires_at、state | view/source/prepare 的持久保留根 | +| `namespace_outbox` | event_id 唯一、seq、view_id、delivery_state | 与 publication 同事务，提交后幂等通知 | + +`selected_ref` 是发布政策，绑定自身始终是 commit/tree。一个 source 的默认 binding 在 v1 只选一个 ref；拒绝多个默认标志的存量异常，不依赖 `.one()` 随机选中结果。以后显式多个挂接位置需要独立政策与冲突规则，不通过现有 repo_path 一对一映射悄悄扩展。 + +数据库结构迁移放 `jupiter-migrate`，通过 SeaORM 工具生成 Callisto entities；按仓库约束，helper 放 `entity_ext`，不手改生成 Model。大规模回填作为可恢复的 application backfill，不塞进自动启动的 schema migration 长事务。 + +字段编码、digest 域分离、nil/空索引、排序及未知字段处理必须在 G01 给出共享 golden vectors 后冻结。当前 JSON/Rust 草图不是任意序列化即可互通的实现标准；canonical bytes 未冻结前不发布 v1 capability。 + +## 6. 发布事务与并发约束 + +已实现的 [publication storage core](namespace-publication-core.md) 包含操作预留/回执、head CAS、同事务 ref 条件写、view/publication/outbox 持久化与两个数据库的故障/并发测试。当前所有生产 writer 尚未接入，组合策略、prepare pin、对象保留与 HTTP 能力仍未完成；下面是完整应用协议，不把存储核心测试等同于全链路交付。 + +统一应用服务接收 `PublishPlan {operation_id, expected_head, ref_read_set, binding_read_set, prepared_objects, native_change?, binding_changes}`，返回 `PublicationReceipt {seq, view_id, outcome}`。HTTP endpoint 名称不决定领域模型；Git 与网页编辑调用同一服务。 + +```text +读取固定 base/head → 构造并验证 candidate objects/index + → 建立 prepare pin，确认对象持久可读 + → 同一个 DB transaction：校验 expected/ref/binding 读集 + + 条件更新 refs/登记政策 + 持久化 scope proof + + 写 view/publication/operation/pin/outbox + + CAS namespace_head + → COMMIT → 回报内容发布成功 → outbox 异步投递 +``` + +这是一个数据库事务，不是两个顺序提交的 ref txn 和 namespace txn。事务内新 tree 元数据可同事务写入，外部 payload 必须在此前持久化；事务失败的 prepared 对象不是已发布数据，prepare pin 到期后才可回收。 + +要求： + +1. `UPDATE ref ... WHERE ref_id = expected_old`；新建用唯一约束防并发重复，删除也验证 expected-old。任何受该发布事务管理的 ref 校验失败，整个事务回滚。 +2. `namespace_head` CAS 同时比较 expected seq/view/epoch，影响行数必须恰为 1。Redis lock 只优化竞争，不是正确性证明；租约锁过期不能让 stale writer 发布。 +3. native candidate 基于哪个 root 构建，必须校验同一 root；网页编辑的 tree 与 commit parent 来自同一固定 base。不能失败后只把 parent 换成 latest 继续写旧 tree。 +4. 失败后重新读取 base、重建计划并重新校验；自动重试只适用于仍满足调用者 expected-old 条件的无冲突操作。用户提交的陈旧修改返回 409，不自动覆盖并发修改。 +5. 同一 push 的多个 branch command 应先计算事务成功后的 ref 集，再按 selected_ref 选择默认 binding；不能把第一条 push command 当全库要发布的版本。 +6. 无可见内容/路由变化时可以只提交 ref/operation；不强制增加 publication_seq。若现有流程确实创建了新的 native root commit，即使 tree 相同，provenance 变化仍按新 view 发布；去除这类额外 root commit 是另一个兼容性优化。 +7. `latest` 从已提交 namespace_head 读取，不拼读几个当前 HEAD。固定 view 读取无需长事务；resolve latest 与创建 pin 需要防止 head 前进后旧 view 被 GC 的竞态。 +8. 响应丢失不代表事务失败；operation receipt 是结果查询依据。outbox 重投不重做 ref 更新。内容已提交而邮件、CL 展示状态或通知失败时，明确 committed 状态，不声称 rollback。 + +PostgreSQL 普通 begin 不自动提供跨多次查询的一致旧快照；在线构造基于不可变旧 view，附加可变读全部进入条件校验。SQLite 使用支持的事务模式/条件写重试，不照搬行级锁 SQL；两个后端都测试。若部署另有对象存储一致性或多个元数据 DB，需重新审查此假设，不能直接开启 capability。 + +## 7. 写入入口覆盖矩阵 + +| 入口 | 需要实施的行为 | 默认 namespace 是否推进 | +| --- | --- | --- | +| 原生 CL merge / 网页编辑最终 merge：`mono/cl/merge.rs::apply_update_result` | 同一 publisher 原子保存 root/path commits、proof、相关 refs；保留 admin-file 检查 | main 可见 root 变化时推进 | +| 原生 scope clone：`transport/pack/monorepo.rs` 与 `code_edit/utils.rs` | 生成派生 commit/ref 时写 scope proof | 单纯 clone 缓存/证明生成不推进 | +| 原生 CL push：`persist_mono_refs`、post-receive | 固定候选 base/scope/head，原有 CL 状态流保持 | 未合并 CL 不推进；生成独立 candidate view | +| 原生路径 attach：`mono/sync.rs` | root 真变化与 proof 纳入 publisher；派生 path refs 同步不能篡改已发布 root | 有可见 root 变化才推进；与后续 merge 是一个还是两个业务发布需明确 | +| import 首次 receive-pack：`transport/protocol/mod.rs` + `post_receive/import.rs` | 提前登记仅 staged；首个有效 selected commit、占位路径与 binding 一起发布；事务内再次检查父子路径冲突 | 内容成功后才新增名字；失败 push 不产生空 binding | +| import 默认/selected branch push | expected-old 校验 + binding 指向新 commit + 原生 attach（若实际发生）同事务 | 推进；旧 view 仍固定旧 commit | +| import 非 selected branch push/删除 | 只更新相应 ref；保留当前默认分支删除禁令并在事务内重验 | 不因该 branch 本身推进；现有额外 root 变更按 §6.6 处理 | +| import 网页 `save_file_edit` | 单 base 构造；objects prepare + selected ref CAS + binding publish | 推进；不允许绕过 D2 不可变策略 | +| import tag REST / Git tag 写入 | ref 与 tag metadata 一致；selector 正确 peel；禁止 snapshot 读回查标签 | v1 固定 commit/selected branch 的 binding 不随 tag 移动；新 resolve 看到新 tag | +| 登记/取消挂接/改路径/换 selected ref、import_dir 变更 | 这是需补齐的管理操作，不声称当前已有完整 API；走 publisher，历史 binding 不变 | 在一个新 view 中原子改变路由 | + +默认分支删除已在 transport 预检查中拒绝；本计划不是补一个“从未存在”的检查，而是把它变成所有相关写入口的事务内约束。移除仓库挂接与删除分支是两种操作，不能自动选一个剩余分支冒充用户意图。 + +Git report-status 维持现有支持的原子/非原子语义，不额外宣称 tags 和 branches 已全批原子化。现有 tag 提前写入需单独回归失败行为；v1 不支持自动跟随 tag 的发布政策，避免把这一差异藏进 namespace 承诺。 + +未来脚本/导入器不可绕开 publisher 直接改活动 refs/registry。上线前用写入口审计与 CI 检查约束低层调用；禁止/隔离未接入的管理写入，再宣告完整 namespace capability。 + +## 8. API 与代码归属 + +遵循 [架构约束](../architecture.md) 和 [Ceres 边界](../../ceres/README.md)：mono 是薄 HTTP router；Ceres 应用服务不依赖 axum/transport 实现；Jupiter 负责存储。新增 REST DTO 在 `ceres/src/model/snapshot.rs`，不是直接塞入 Orion 的 `api-model`。 + +| 拟改动位置 | 责任 | +| --- | --- | +| `ceres/src/application/snapshot/{resolver,publisher,bindings,lease,mod}.rs` | 独立 SnapshotApplicationService；通过对象/存储 port 复用现有能力 | +| `ceres/src/application/api_service/mono/app_services.rs` | 注入并提供 snapshot 服务 accessor，不继续扩张 MonoApiService 为全部实现容器 | +| `jupiter/src/storage/snapshot_storage.rs` 与 storage mod | 事务内条件写、view/index/operation/pin/outbox；无需 application 反向依赖 | +| `mono/src/api/snapshot.rs`、router、`api_doc.rs` | 鉴权 context、DTO→领域调用、统一错误、utoipa 注册 | +| `common/src/errors` 现有错误定义/转换 | typed snapshot errors 与 HTTP 映射，不新增字符串 `[code:...]` 协议 | +| `jupiter-migrate/src/migration` 与 Callisto | additive schema + generated entities +回填状态 | + +API 路径是候选设计，实际 OpenAPI 由 Rust/utoipa 生成；不维护一份与代码竞争的手写 OpenAPI 文件。 + +面向大量源码小文件的目录分页、按需 tar + zstd 小包、热点包复用、大文件分块及客户端调度细化见 [文件传输协议 v1](scorpiofs-transfer-v1.md)。这是尚未实现的传输扩展，不修改 NamespaceView 身份或现有单 source 完整对象契约。 + +| 拟议 API | 必须保证 | +| --- | --- | +| `GET /api/v1/snapshots/capabilities` | instance、schema、算法、路径编码、source/namespace readiness 与 retention 限制 | +| `POST /api/v1/snapshots/resolve` | typed selector；原子获得固定 descriptor + pin/lease；明确 consistency | +| `GET /api/v1/snapshots/{id}` | 相同 ID 的规范 descriptor 不变；可用性/租约另作 envelope | +| `GET /api/v1/snapshots/{id}/bindings` | 固定 prefix/cursor，惰性读取受限索引 | +| `GET /api/v1/snapshots/{id}/tree` | 固定路由；entry 有 type/mode/OID/source、准确 size 或后续同对象 stat token | +| `GET /api/v1/sources/{id}/trees/{oid}` 与 `/blobs/{oid}` | 明确 source、类型、算法、snapshot 授权上下文；tree 原始字节可校验 Git hash | +| `POST /api/v1/snapshots/{id}/leases`、`DELETE /api/v1/snapshot-leases/{id}` | 幂等创建/续期/释放；期限使用服务器时间 | +| `GET /api/v1/snapshot-operations/{operation_id}` | actor-domain 限定结果查询，恢复丢响应；不可跨用户枚举 | + +固定树 entry 若返回未知 size，ScorpioFS 必须在向 FUSE stat 报告前取得该 OID 的精确长度；不能用 0 占位。二进制 blob 返回文件原字节，Git 哈希按类型+长度头校验，不能错误剥掉文件本身的相似前缀。 + +## 9. 授权、保留与故障语义 + +**D4 已于 2026-09-06 获用户确认**：snapshot 读 API 默认关闭，只有显式配置 source/scope 读授权和对象保留策略后才允许启用。当前基线通用 Cedar guard 主要覆盖 CL，并有开发期 permit-all 策略；不能把接入该 guard 当成满足本门槛。配置缺失/无效、授权或保留实现未就绪时拒绝开启 capability；当前基础实现尚未暴露 snapshot HTTP 路由。具体 ACL/lease 配置与实现仍需在后续 API PR 中验证。 + +全局 object store 命中不是访问授权。snapshot lease 只保留数据，不赋予永久读权；鉴权仍用当前访问政策，撤权可返回 403，但不得换成 latest 的内容。descriptor/binding 分页也可能泄露私有路径，不能只保护 blob。 + +读请求携带由固定 tree walk 产生的 object ticket 或等价可验证上下文，限定 source、scope/root、类型与 OID；初始 root ticket 由 resolver 生成。客户端随意填一个存在的 OID 不证明其可达性。优先采用可惰性展开的路径/父 tree 证明，不为每个 mount 扫描整个可达闭包；ticket 防伪、过期与续期机制必须与 lease 配套，不能当作绕过当前 ACL 的能力令牌。 + +GC roots 至少包含有效 publication 保留窗口、source/view leases、prepare pin、候选 CL pin。以树/对象图进行标记，lease 创建/续期与回收共享 GC epoch 或等效协调；不得在检查过期后与续期竞争删除有效对象。repo tombstone 不能级联删掉旧 view 仍引用的对象。 + +若按 pack 删除，包内任一保留对象要求保留整个 pack 或先安全 repack；若有 deltified 对象，还必须保留解码所需的 base 链。只标记直接 tree/blob OID 不足以证明底层 pack 可删。未完成 Git 对象/存储层保留审计前，可采用明确配置的“不回收这些对象”试点，但不能宣称已实现有界 GC。 + +错误 envelope 为 `{code,message,retryable,details}`；公开响应避免泄露跨权限域对象是否存在。 + +| HTTP / code | 语义 | +| --- | --- | +| 400 `INVALID_SELECTOR/PATH` | 输入类型或规范路径不合法 | +| 403 `FORBIDDEN` | 当前授权不允许；如采用隐藏存在性策略可统一为 404 | +| 404 `SOURCE/OBJECT/PATH_NOT_FOUND` | 授权域内不存在；空目录是成功的空 entries | +| 409 `EXPECTED_VIEW_MISMATCH/REF_MOVED/BINDING_CONFLICT` | 条件不满足，不静默重基 | +| 409 `SOURCE_SCOPE_MISMATCH/SOURCE_SCOPE_UNVERIFIED` | 错误或无法证明的 commit scope | +| 409 `IMMUTABLE_BINDING/DEFAULT_REF_REQUIRED` | 违反已选择的发布政策 | +| 410 `SNAPSHOT_EXPIRED` | 对象可用性不再承诺；仅 manifest 尚存不是有效租约 | +| 422 `HISTORICAL_BINDINGS_UNAVAILABLE` | 旧主仓 commit 没有对应历史 catalog | +| 501 `CAPABILITY_UNSUPPORTED` | 如不支持的算法、非 UTF-8、LFS hydrate | +| 503 `OBJECT_UNAVAILABLE/PUBLICATION_NOT_READY` | 暂时不可读或未就绪；没有 latest fallback | + +服务端对象错误映射成客户端 I/O 错误与独立诊断，不得伪造空文件/空目录。租约时长与历史保留期是部署参数，在实施前确认；这份 spec 不擅自承诺永久保留。 + +## 10. 迁移、开启与回退 + +1. **Additive schema**：只建新表/索引，旧读取继续。迁移前备份与校验，按 [migration workflow](../../jupiter-migrate/README.md) 测 PostgreSQL/SQLite 升级；生产回退不 drop 新历史表。 +2. **Source capability**：按源历史读与 scope proof 到位后可独立开启；未具备完整路由历史就不返回 namespace capability。 +3. **Writer fencing**：全部服务实例升级到能遵守 publisher 的版本；短期阻断旧 writer/管理脚本。writer_epoch 只有所有入口执行检查才有效，不能把新列本身当作旧二进制已被隔离。 +4. **初始 catalog**：推荐第一版维护窗口内暂停有关元数据写入，获取一致 native root/registry/default refs，验证对象和 scope、构建索引并保存可恢复回填进度；遇无默认 ref、重复默认、嵌套冲突或对象缺失，列入异常清单，不发布缺项“完整”视图。 +5. **Cutover**：在仍受写屏障保护时发布 seq 初始值与完整 view，切换写入口后解除屏障；旧 API 可继续提供 latest，但新 ScorpioFS 只使用 snapshot API。大规模回填过长时另设计在线 changelog/catch-up，不能只用普通事务分页扫描冒充一致快照。 +6. **Shadow/read compare**：固定时间点对比独立物化 oracle，不将活跃 latest 的漂移误判为快照错误。后台可以审计 head/ref/binding 一致性，但后台补偿不是原子发布的替代品。 +7. **Rollback**：先停新发布，保留既有固定 view 读取和 leases；回退的服务若会绕过 publisher 写入，必须停相关写流并撤下 namespace latest capability。禁止静默用旧实时路由服务已分配的 view_id。 + +历史起点必须公开：初始 seq 之前缺 catalog 的主仓 commit 仍只能读 native scope，或由用户提供显式 bindings 构造 `consistency=explicit_composition`；后者可重放，但不是补回了当年的全库原子快照。 + +## 11. 第一轮 PR 切片与退出条件 + +下列是实施工作包，不是已创建的 Mega issue/PR；不提供未经排期的完成日期。 + +| 包 | 交付范围 | 退出门槛 | +| --- | --- | --- | +| G01 契约/fixture | typed selectors、路径规范、canonical bytes/golden vectors、固定 native/import A/B 场景；两仓共同消费 fixture | digest 跨语言一致；歧义 selector/错误 scope 拒绝；D1 影响功能分级 | +| G02 单 source 历史读 | Import refs 修复的新 resolver、scope-aware tree/blob、权限与错误；scope proof 的最小存储 | MG01–MG04/MG13；旧 API 回归；不宣称 namespace 已完成 | +| G03 发布存储核心 | additive schema、bounded binding trie、publish CAS/read-set/receipt/outbox、pins | MG05/MG06/MG09/MG12/MG15；两个数据库后端 | +| G04 接入全部 writer | §7 矩阵：native merge、import push/web、建仓 staging、政策变更与 proof | MG07–MG11；每个生产入口均有覆盖，无后台补偿窗口 | +| G05 迁移/租约/能力上线 | 回填、writer fencing、保留根与对象存储审计、capabilities、故障演练 | MG14–MG17；保留参数明确，完整性审计通过 | +| G06 ScorpioFS 联调 | 真 Mega 双版本挂载、分页、惰性旧 import、并发发布、租约错误传递 | ScorpioFS V01–V09/V14/V15/V17 与独立 oracle；无 FUSE/有 FUSE 分层 | + +G02 可以与 ScorpioFS fake backend/CAS 类型工作并行，G03/G04 不等“共享 lower 架构选型”。先完成一个 native + 一个 import 的真实端到端切片，再扩规模，避免只做大量表和 endpoint 而未证明历史读取。 + +## 12. 验收与故障注入 + +以下 MG 是完整交付的验收定义，不能将基础单元测试等同于整项通过。已验证的索引/数据库子集见 [namespace-index-v1](namespace-index-v1.md)：包含百万 binding 的内存索引门槛和 PostgreSQL/SQLite 持久化、回滚及 UTC 升级测试；MG12/MG15 仍只有部分覆盖。完整 native/import composition、publisher 与 FUSE 验收尚未运行。fixture 的期望内容由独立 Git object 物化与显式 binding composition 得到，不用被测 resolver 生成期望值。 + +| ID | 场景及断言 | +| --- | --- | +| MG01 | import branch 从 A→B，固定 A 的 tree/blob/size 永远是 A；同路径 A/B 同时可读 | +| MG02 | native root/scope 同路径寻址、同 commit 多 scope、缺证明、清理 child ref；证明不丢且不重复拼前缀 | +| MG03 | annotated/lightweight tag、同名 branch/tag、tree/blob target；解析只做一次且错误明确 | +| MG04 | 空目录、缺目录、旧 oid、损坏 Git bytes、symlink/mode；不能以空成功代替错误 | +| MG05 | native merge 与 import update 并发：线性化结果是两个可串行化的新 view，不丢任一成功更新 | +| MG06 | same ref 两个 expected-old writer：最多一个成功；陈旧网页编辑不把旧 tree 接到新 parent | +| MG07 | 多 branch push 的 selected ref、非默认 push、默认删除、失败 tag/branch 混合；按承诺返回，不任取第一条 command | +| MG08 | 新 repo 登记后 unpack 失败、并发父子仓登记、取消挂接/改路径；失败新 repo 不出现在发布目录，旧 view 路由不漂 | +| MG09 | objects prepare、ref write、view write、head CAS、commit、response、outbox 各处 crash；已提交结果可查、未提交不出现 | +| MG10 | 网页编辑/native merge/管理入口的绕过测试；全部可见 mutation 都能查到对应 publication 或明确 no-op | +| MG11 | D2 已确认：显式 release 目录拒绝所有写入口的第二次内容变更，不能通过降级 policy 绕过；普通 mutable 开发目录更新时保留旧绑定/对象 | +| MG12 | 百万 binding 单点更新、小前缀 mount、跨多页目录：节点与内存有界，不扫描整个 registry，cursor 不串 view | +| MG13 | 猜测另一个 repo 的 OID、已撤权 lease、私有 binding、跨域 CAS 命中；均不能绕过授权 | +| MG14 | pin 创建/续期与 GC 竞争、pack/base 保留、repo tombstone、lease 过期；不删有效保留对象 | +| MG15 | PostgreSQL/SQLite 同样的事务失败、竞争、幂等 key 异 payload；无仅在某后端成立的保证 | +| MG16 | 回填中断续跑、缺默认 ref/对象、旧 writer、首次发布窗口和回退；不误宣布 capability | +| MG17 | 初始 seq 前的历史 root 缺 binding catalog：422；显式组合标注非原子历史；后续首次惰性读取仍固定旧 import | + +发布指标最少包含 publish_conflict/retry、prepared bytes、txn duration、outbox lag、active pins、retained bytes/pack amplification、index nodes read/written；标签不加入无界 path/OID。 + +## 13. 待确认与实施前闸门 + +产品决策仍沿用客户端 spec 编号，避免两仓各自解释： + +- **D1 已确认（2026-09-06）**：Mega 原子发布 native root + 固定 import bindings 的完整 namespace view，ScorpioFS 固定该 view 读取。单 source 是中间工作包，不将全库原子一致性从本次目标延期。 +- **D2 已确认（2026-09-06）**：明确标记为 release 的目录首次发布后不可改，通用 import 开发 branch 仍可演进；不能仅从数字路径名自动判断政策。新 release 内容使用新版本路径，所有写入口都必须执行约束，不能通过改 policy 绕过。 +- **D3 推荐**：运行 build 固定旧 view，新 build 用新 view；现有工作区受控切换。透明 live-refresh 不属于本服务端 PR 的承诺。 + +实施闸门：G01 冻结 canonical 编码/字段；G04 审计全部写入口；G05 明确实际 PostgreSQL/SQLite、对象后端、导入拓扑、维护窗口及保留期。基础设施验证可继续，不把这些未确认事项写成“用户已同意”。 diff --git a/docs/spec/scorpiofs-transfer-research.md b/docs/spec/scorpiofs-transfer-research.md new file mode 100644 index 000000000..5cd937e9f --- /dev/null +++ b/docs/spec/scorpiofs-transfer-research.md @@ -0,0 +1,235 @@ +# ScorpioFS 传输研究：问题、机制与证据计划 + +状态：研究设计草案，2026-09-06。没有论文性能结果，也没有已证明的新颖性结论。 +本文补充 [协议 Spec](scorpiofs-transfer-v1.md)，面向 Mega 服务端与 ScorpioFS 客户端共同实现。读者入口仍是 [通俗设计](../scorpiofs-transfer-design.md)。 + +## 1. 论文究竟要回答什么 + +候选研究问题： + +> 在主仓与挂载仓库组成的版本化 monorepo 上，多个短生命周期 workspace 面对不同版本、部分缓存和不断出现的文件需求时，如何在可重放读取的约束下，降低首次有效构建和后续更新的输入准备成本？ + +其中，文件传输子问题是： + +> 根据当前需求、本地缺失对象、可复用的小包和服务端压力，何时逐文件读取、何时临时打包、何时下载现成包？ + +这不是承诺所有负载都受益。低 RTT、少量串行读取、压缩服务饱和、低缓存命中、频繁目录修改和不准确预取都可能使打包变慢。系统必须能退回逐对象读取，实验必须包含这些情况。 + +暂定论文主线是固定版本上的共享、按需 workspace；传输是其中一个可独立消融的机制。是否足以成为独立协议论文，需要文献比较与实测后决定，不能提前把每个模块包装为一个贡献。 + +## 2. 已有工作告诉我们什么 + +本轮是围绕机制的定向检索，不是穷尽式综述。以下来源为会议论文、项目官方文档和官方协议；没有用引用数推断技术优劣。 + +| 工作 | 年份/类型与证据 | 已有能力或研究问题 | 对本设计的要求 | +| --- | --- | --- | --- | +| [Slacker: Fast Distribution with Lazy Docker Containers](https://www.usenix.org/conference/fast16/technical-sessions/presentation/harter) | FAST 2016；会议摘要 | 利用共享后端和惰性获取降低容器准备成本 | “不先下载完整内容”不能作为新颖性 | +| [DADI: Block-Level Image Service for Agile and Elastic Application Deployment](https://www.usenix.org/conference/atc20/presentation/li-huiba) | USENIX ATC 2020；摘要与公开论文 | 以细粒度按需传输替代镜像下载/解包的串行准备 | 必须比较按文件与按块加载；不把容器负载结论直接移植到源码构建 | +| [Towards a responsive CernVM-FS architecture](https://cds.cern.ch/record/2701496) | 2019，EPJ Web of Conferences 214, 03036；论文页及公开摘录 | 版本目录、内容寻址与 HTTP 按需软件分发 | 版本、目录元数据和 CAS 各自已有先例；不能仅凭系统组合宣称首次 | +| [FlacIO: Flat and Collective I/O for Container Image Service](https://www.usenix.org/conference/fast25/presentation/liu-yubo) | FAST 2025；[公开论文](https://www.usenix.org/system/files/fast25-liu-yubo.pdf) 的摘要、引言及机制概览 | 利用 trace 生成服务相关 runtime image，以聚合传输和运行时缓存减轻惰性加载开销 | “根据工作集聚合传输”已被研究；必须检验新版本、部分缓存、预测失效和准备成本 | +| [EdenFS Overview](https://github.com/facebook/sapling/blob/main/eden/fs/docs/Overview.md) | 工业系统官方文档，核对于 2026-09-06 | 面向大规模源码仓库的惰性文件系统 | 是接近实际问题的系统比较对象，不能只比完整 Git clone | +| [Remote Execution API](https://github.com/bazelbuild/remote-apis/blob/77ec630134abbf9aa525f921eee4e5d11dc20f7e/build/bazel/remote/execution/v2/remote_execution.proto) | 官方协议，固定所查 revision | CAS 批量 blob 读取、ByteStream，以及 SplitBlob/SpliceBlob 定义 | 批量读取和文件分块都不能单独作为新协议贡献；需要强批量 CAS 基线 | + +这张表只记录读到的机制，不表示其他系统缺少某项能力。正式 related work 需要补读全文、实现和近年后续工作；尤其不能根据摘要写“其他系统不能处理多版本”。 + +候选贡献必须落在可验证的机制与结果上：组合版本约束下的可重放共享、针对缓存与版本变化的传输选择，以及完整服务端/客户端成本证据。若实验发现固定批量策略已经足够，应保留简单方案，并收缩自适应算法的贡献表述。 + +## 3. 在选参数前先做工作负载测量 + +不能因为文件多，就假设它们会被同时读取。先从至少三个不同工具链的公开源码构建中记录： + +- 文件大小分布；metadata-only 操作与真正 read 的比例。 +- 相邻访问是否在同一目录、同一 target、同一 source；每次有多少独立未完成读。 +- 首次成功编译动作与整个构建分别需要哪些文件；依赖链有多少串行 miss。 +- 连续真实 commits 中，需求对象集合和内容 ID 的重合度；新旧路径相同不等于内容相同。 +- 同主机多个 workspace 的需求重叠；多主机之间的重叠另算。 +- Mega 原始对象读取、对象长度查找、压缩、队列等待及客户端解压/哈希的成本。 +- 初次发布目录索引、预制包、分块表的时间与空间；复用多少次才能摊销。 + +至少区分三种信息条件： + +1. **纯按需**：只知道已经发生的 FUSE 需求，不知道后续文件。 +2. **合法提示**：使用执行前已经存在的构建输入清单或之前运行的 trace；获取提示的成本计入结果。 +3. **离线 oracle**：知道本次完整未来 trace，只作为参考下界，不能成为在线算法输入。 + +一个目录有 1,000 个文件并不意味着客户端能自然聚合为 8 个请求。只有这些需求已并发出现或已由合法提示给出时，128 个一包的算术才成立。串行依赖、跨 source 分组和包大小上限都可能增加请求数。 + +## 4. 问题模型与正确性不变量 + +版本 v 的固定路由为 R(v, p) = (source_snapshot, relative_path, kind, oid)。 +客户端当前可复用的已验证对象集合为 C。时刻 t 已出现的需求集合为 D(t)，待下载对象为 D(t) 减去 C 和可加入等待的 in-flight 对象。 + +R 是 namespace 的语义；包 P 只是固定对象的运输集合。修改包的大小、排序、编码或缓存布局不改变 R。 + +需要验证四项不变量： + +- **版本一致**：所有路径解析使用已固定 v；任何 pack miss、重试、hint 失效和缓存淘汰都不重新解析 latest。 +- **内容一致**：完整 blob 通过 Git 类型/长度/哈希验证；大文件块依赖已确认可信的 Mega 分块表，并验证块摘要。两者的保证不混为一谈。 +- **权限隔离**：每个请求中的路径先绑定到 v 和当前 source/scope 权限；命中内容缓存或包缓存不是授权证明。 +- **故障不伪造数据**：未完成对象不对读者可见，缺失对象不以空文件替代；同一对象的 waiter 独立取消，旧版本响应不填充新版本路径缓存。 + +正确性论证草图：以 Mega 的原子发布与固定路由为前提；每次读取由 R 得到预期对象，再验证从单对象、包或分块得到的字节，最后仅向对应版本的 waiter 返回。改变运输集合不能改变期望对象。这个论证还需要生产 writer 覆盖、状态机测试和真实 FUSE 验证支撑，不是已完成的形式化证明。 + +建议的最小状态模型为 (published_view, workspace_view, expected_oid, transfer_state, cache_state, permission_epoch, lease_state)；枚举 publish、lookup、join、cancel、deliver、corrupt、expire、crash。目标包括旧版不漂移、半对象不可见、一个取消不伤其他 waiter。有限模型不能证明真实网络和存储实现正确。 + +已有数据进入进程或 kernel page cache 后无法瞬间撤回；论文明确在线授权与数据交付边界。可信 Mega 分块表也不等价于仅凭 view hash 验证一个恶意服务端。 + +## 5. 传输选择策略：可解释、可替换、可消融 + +先保留三个可实现动作： + +| 动作 | 适用条件 | 必须计入的代价 | +| --- | --- | --- | +| 并发逐对象读取 | 少量或紧急需求、没有可靠提示、服务端压缩忙 | 每请求处理、并发网络、单对象存储和校验 | +| 精确缺失列表包 | 多个已知小文件缺失，预打包会重复下载 | 构包排队、读取全部成员、压缩、首文件等待 | +| 已缓存局部包 | 包已存在，很多内容确实需要，过量传输可接受 | 已缓存/不会用的额外字节，解压全部前缀与成员顺序 | + +所有候选先经过版本、source/scope、权限、协议上限和客户端资源检查。source 边界由协议固定；不会为了压缩率放宽授权。 + +在线状态不需要加入 NamespaceView，可以放在客户端调度器和 Mega 的派生包目录： + +~~~text +Demand { + workspace, view, source, path, oid, size, + arrived_at, deadline, priority +} +Candidate { + mode, covered_demands, members, cached_pack_id?, + estimated_wire_bytes, estimated_build_ms, + estimated_decode_ms, estimate_confidence +} +SchedulerState { + verified_objects, inflight_objects, + data_slots, reserved_demand_slots, + memory_budget, prefetch_budget, + recent_rtt, recent_goodput, recent_pack_cost +} +~~~ + +评估的是完成同一批已知 D(t) 的完整计划，未被某候选覆盖的需求也必须加入代价，不能让“只完成一个最快文件”的方案虚假获胜。 + +一个候选包中对象 i 的粗略完成时间估计： + +~~~text +T_i = 等待聚合和客户端槽位 + + 网络往返 + + 服务端排队与准备整个包 + + 接收至成员 i 结尾的网络时间 + + 解压并验证该前缀的时间 +~~~ + +服务端当前小包契约在发送前构造完整包，所以构包成本必须加在首对象完成时间之前。规范中按 OID 排序，紧急文件可能在包尾；这种情况下可将它单独请求，不通过未协商排序修改包格式。 + +这个加法是保守的预测特征，不是吞吐定律：网络、解压可能流水重叠，多个流争用相同带宽，必须根据历史样本校准。逐对象基线也使用同样的连接并发，不能使用 N × RTT 作为它的预测成本。 + +初版选择器： + +1. 剔除缓存命中，加入已有 in-flight；已阻塞且接近 deadline 的读优先单独调度。 +2. 最多考察 128 个待处理对象及有界包提示，按 source/scope 分区；服务端 hint 是性能建议，客户端仍逐项验证可用性。 +3. 生成上述三类可行完整计划，用最近的 RTT、goodput、实际 pack 构建时间与现成包 wire size 预测需求完成时间。缓存包没有构建成本，新包不能假设零 CPU。 +4. 在内存、数据流、预取浪费和 demand 等待约束内，选择预测平均需求完成时间较小的计划；限制人为聚合等待并为旧请求保留执行机会，避免用少量长尾换平均值。这不保证网络失效或过载时仍能在 deadline 内完成。 +5. 估计样本不足时采用无预取的保守按需基线。实际完成后更新同编码、大小桶、服务端域的成本统计，记录选择原因和预测误差。 +6. 聚合窗口、75% 缺失阈值、包数和字节上限均有静态策略对照。在线策略不能在等待聚合期间阻塞唯一可能产生下一个需求的线程。 + +这是拟议的受限启发式，不声称全局最优或具有竞争比。参数、估计方法和选择代码需在测试集实验前冻结；若在线策略收益不足以覆盖自身开销，则移除它。 + +首轮 grouping 保持同目录、稳定名称分组。后续仅在 trace 显示大量跨目录共访时，研究同一 source/scope 内基于历史共访的小包：时间窗口构图、成员数/字节约束、构建预算均需单独定义和消融。不能把当前目录包描述为已经实现的语义或依赖图打包。 + +## 6. 服务端成本与复用的真实边界 + +包缓存单位由对象集合及编码确定，客户端缓存单位为 blob，目录绑定由 view 确定。三者分离后，新版本中相同对象可直接复用；包分组变化不强迫客户端重下未变文件。 + +但是，这并不免费： + +- 每个定制缺失集合都可能生成不同包，造成服务端缓存碎片。记录包复用次数和一次性包比例；一次性包只进行短期 in-flight 合并，不无界持久保留。 +- 命名稳定的目录分组在插入/删除文件后也可能重新分组。测量新包构建字节与真正变更字节之比，不能仅展示客户端下载减少。 +- 身份、权限、size 和历史路径验证仍有成本。服务端将公共路径前缀及 OID 查询合并，但不能略去这些检查以赢取基准。 +- 多 workspace 同主机可共享内容缓存和 single-flight；多主机默认仍各自下载相同内容。服务端 pack cache 只省构包 CPU/后端读取，不等价于跨主机网络广播。 +- 一次构包或物理 fetch 计一次；多个 waiter 的逻辑等待分别计数。取消首发请求不应破坏其他需求。 +- 预制索引、小包或大文件块的维护成本既单独报告，也在短生命周期场景计入端到端成本。给出复用 1、10、100 次的摊销曲线，不能只测无限热缓存。 + +## 7. 可证伪假设与退出规则 + +| 假设 | 需要的证据 | 何时收缩或否定 | +| --- | --- | --- | +| H1：传输选择对不同 RTT/缓存/负载优于固定策略 | 相同并发及编码下，比最佳预先调好的静态策略改善有效动作与完整构建时间，且邻居尾延迟不过度恶化 | 只有比串行 GET 才快，或收益只来自 zstd，则不能宣称调度创新 | +| H2：跨版本对象复用抵消包分组变化成本 | 连续真实 commits 中客户端下载、服务端重构、元数据与索引开销完整计量 | 节省网络却造成服务端 CPU/延迟更大，不写总体更高效 | +| H3：固定版本下共享 fetch 能降低多 workspace 的重复成本 | 1/4/16/64 workspace，同版与混合版都保持正确树，物理/逻辑计量分开 | 混合版本破坏隔离、资源持续增长或单租户受严重干扰，则机制不通过 | + +这些是研究假设，不是已经达到的验收结论。主论文可以只保留得到支持的假设。没有效果或退化的实验同样保留，不能更换指标后隐藏失败。 + +## 8. 比较对象与公平条件 + +协议基线至少包括： + +- 并发 HTTP/2 逐对象读取，同样的目录元数据、CAS、鉴权、租约和连接预算。 +- 固定大小批量读取，分别使用无压缩和 zstd；为新选择策略提供真正同条件的对照。 +- REAPI 风格的 BatchReadBlobs + ByteStream。Git blob OID 与裸 payload digest 不是同一个定义；需要明确桥接并计入代价。若未运行真实 REAPI 实现，只能称为机制模拟，不能声称战胜 Bazel。 +- 固定目录小包，以及已有历史 trace 的工作集包。后者用来检验“已知工作集直接打包”是否已经足够。 + +系统基线包括原有路线中的 Git full/partial+sparse、bare+worktree、snapshot+CAS、共享 fetch，并增加与实际可运行的 EdenFS、CernVM-FS 或现代惰性镜像系统的适用性核查。 + +比较不能改变要构建的内容：所有基线物化同一个 native+imports 版本；不具备组合 namespace 的系统由实验准备器生成等价只读树，准备成本单列。若通过适配层导致能力不等价，必须在结果中标注限制。 + +容器系统与源码工作区用途不同，只有在相同文件树、目标任务、缓存状态和正确性条件下才能做数值比较。源码/权限/运行条件不满足时保留机制比较，不能用简化模拟冒充原系统实现。 + +首轮直接服务端传输基线固定 CPU、磁盘和网络预算；后续可单独评估 CDN/代理，并把授权路由与带宽计费变化列出。不能让一个系统用预热代理而另一个直连冷后端。 + +## 9. 实验设计与数据口径 + +先固定 workload、commit、目标、工具链、机器、内核、FUSE 模式、网络与缓存初始化，复用 ScorpioFS 的 M0 harness。WSL 只做功能与小规模 smoke，主结果采用可控 Linux 机器。 + +主结果至少包含：新 workspace 到首次成功构建动作的时间、完整构建时间、构建成功率；传输时延和请求数用于解释原因。计时起点必须包含 resolve、lease 和必要元数据，不只从下载 body 开始。 + +首次有效动作的具体命令/target 和成功判据在实验 manifest 中固定，不能事后挑最快完成的编译单元。每个基线先用独立 oracle 检查 path/type/mode/content；构建输出采用固定工具链下预先规定的正确性检查，无法逐字节重现的产物单独说明。 + +### 实验轴 + +- RTT 1/20/80 ms；带宽 100 Mbit/s 与 1 Gbit/s；补充受控丢包与带宽突变。 +- 缓存全冷、目录热内容冷、内容热 kernel 冷、全部热；Mega 包缓存冷/热独立控制。 +- 同版/混合版；真实连续 commits，另用 0/1/10/50% 变更率的合成数据定位边界。 +- 少量串行文件、很多并发文件、跨目录稀疏访问、metadata-heavy、含大文件混合构建。 +- 同主机 1/4/16/64 workspace；多主机另一个实验，不混淆共享域。 +- 服务端低/高压缩负载，冷热 Git pack，包缓存容量和淘汰压力。 + +### 消融 + +1. 单对象 → 固定批量，隔离批量收益。 +2. tar → tar.zstd，隔离压缩收益。 +3. 无包缓存 → 有包缓存，隔离服务端复用收益。 +4. 固定选择 → 估计成本选择，隔离算法收益。 +5. 无预取 → 目录提示 → 历史 trace 提示，隔离信息与预取收益。 +6. 无跨 workspace 合并 → single-flight；同版与混合版分别测试。 +7. whole-blob → chunk reader，分别报告随机 read 放大和顺序吞吐。 + +包大小、并发、等待窗口及预取阈值先在训练/调优子集搜索，再冻结并用于留出的项目和后续 commits。相同程序运行的测试 trace 不得反过来构造它自己的在线预取包；offline oracle 单独标记。 + +### 字节和时间 + +- encoded_body_bytes：实际传输的编码后 body，包含压缩/包头/重试。wire_bytes 另外通过 socket/网络测量统计 HTTP/TLS 等开销并声明计量层，不能把 body 大小冒称完整网络流量。 +- fetched_raw_bytes：解压后实际运送的对象内容，重复下载重复计。 +- useful_bytes：固定观察窗口内被 demand 实际消费的唯一字节范围；同时报告唯一被用到的完整 blob 大小，区分文件级与范围级读放大。 +- unused_prefetch：该窗口内未被 demand 使用的预取内容；重复命中已有缓存的传输另记 duplicate_bytes，不能藏在压缩率里。 +- logical_requests、physical_fetches、pack_builds、DB queries、disk IO、server CPU、client CPU、RSS 和持久化存储分别记录。 +- pack 等待、构建、first-byte、first-verified-object、all-demand-ready 分开。一个包 HTTP 请求结束更早不一定让关键编译动作更早完成。 +- 浪费率分母为实际预取字节，零预取时记 N/A；同时报告浪费字节对 demand 字节之比。所有观察窗口长度必须固定。 + +主结果至少 10 次独立运行，随机化方案顺序，保存全部失败、超时及原始样本；按独立运行做区间估计，不能把同一构建的千个相关读当千次独立试验。P99 需要足够独立负载与操作样本，不能由十个端到端样本硬推。 + +报告绝对值、相对值、置信区间和每 workload 结果。若使用综合分数，事先声明权重/几何平均定义；不能只选最好的一组 RTT。队列压力下报告租户 slowdown 分布，而不只给总吞吐。 + +## 10. 先做什么，以及论文能写到哪里 + +下一步先交付测量与正确读取的小切片： + +1. 将 M0 的事件关联到 view、source、包、对象、waiter 和真实物理请求；固定公开两版本 fixture 与独立 Git oracle。 +2. 建立并发逐对象、固定批量和 tar.zstd 基线，端到端跑至少一个真实构建。 +3. 采集缓存/版本/并发矩阵，检查确实存在固定策略的失败区间。 +4. 只有发现这种区间后，再实现成本选择器；始终保留简单静态策略可用。 +5. 接入真实原子 namespace 发布、租约和双版本 FUSE 验证，再扩大系统实验。完整论文结论不由库层 codec 单测替代。 + +协议实现、算法原型、研究主张和测量结果分别维护状态。本次完成的是研究设计与相关工作初筛，未运行实验。 + +论文结构建议：工作负载证据 → 版本与共享约束 → 传输机制/选择策略 → 正确性与信任边界 → 分解实验和端到端结果 → 失败区间/局限 → 相关工作。 + +如果最终只是 tar.zstd 小包比逐文件下载更快，可以作为系统实现中的优化报告；要将其作为核心贡献，需要证明相对已有批量 CAS、工作集聚合和版本化惰性文件系统，新增机制在明确条件下提供可复现的收益。 diff --git a/docs/spec/scorpiofs-transfer-v1.md b/docs/spec/scorpiofs-transfer-v1.md new file mode 100644 index 000000000..e200e4411 --- /dev/null +++ b/docs/spec/scorpiofs-transfer-v1.md @@ -0,0 +1,401 @@ +# Mega → ScorpioFS 文件传输协议 v1 + +状态：Draft，2026-09-06。本次仅设计，未增加 HTTP endpoint 或已部署 capability。 +先读 [带实例的传输设计](../scorpiofs-transfer-design.md)。 + +论文研究问题、相关工作、候选选择算法、正确性不变量和公平实验要求见 [研究设计](scorpiofs-transfer-research.md)。本文定义运输契约；其中 75% 缺失阈值、1 ms 聚合窗口和包大小是静态基线参数，不作为新颖性或普遍性能保证。 + +已确认：以大量源码小文件为主，支持局部打包；大文件由 Mega 校验完整 Git blob 后生成分块表，客户端信任经认证的 Mega 所提供的对应关系。保留既有默认关闭、显式授权和对象保留门槛。workspace 如何切换版本仍由客户端 spec 定义，本协议不依赖透明切换。 + +## 1. 与已有实现的关系 + +当前 ScorpioFS [SourceReader](https://github.com/gitmono-dev/scorpiofs/blob/codex/system-paper-spec/src/snapshot/backend.rs) 逐层获取 tree，并通过 Bytes 返回整个对象，默认 tree/blob 上限分别为 16/64 MiB；它尚无批量对象协议和随机分块读取。旧 Dicfuse 的 inode 内容缓存也不能直接充当跨版本的对象缓存。 + +Mega 已有按父目录合并 blob 查询的 [blob_ops](../../ceres/src/application/api_service/blob_ops.rs) 和按仓库批量查询对象元数据的 [GitDbStorage](../../jupiter/src/storage/git_db_storage.rs)，可以复用查询思路。当前 helper 的实时路由、出错跳过和 Vec 全量读取行为不能直接成为这里的契约。 + +本协议新增传输 DTO 和可重建缓存，不向严格编码的 SourceSnapshot、NamespaceBinding 或 NamespaceView 中添加字段。包 ID、压缩方式、分块表不参与 view_id。相同版本使用不同压缩器仍表示相同文件。 + +## 2. 协议选择与边界 + +- 传输：HTTPS，优先 HTTP/2；允许 HTTP/1.1 连接池保持相同语义，未协商 h2 时不能声称获得 h2 性能。 +- 元数据：UTF-8 JSON；字节大小和 offset 使用无符号整数，v1 最大对象 8 TiB，以 capabilities 声明的更小限制为准。 +- 小文件批量：标准 ustar + 单个 zstd frame；不使用跨包字典、Git delta、Base64 或自定义二进制帧。 +- 单个文件与分块：二进制 HTTP body。内容字节与压缩字节分别计数。 +- source、path、OID、模式和错误规则继承既有 snapshot spec。v1 不展开 gitlink 或 LFS 指针，不自动跟随远端 symlink。 +- JSON 示例中的 V100、A、P 等是标注过的教学简写，不是可通过严格 ID 校验的 fixtures。 + +Git pack 支持 delta/base 链，适合 Git 对象传输；本设计选择独立小包以限制解码依赖和重试单位。以后若实测证明 pack 更优，可以协商另一种编码，不把它作为首版依赖。[Git pack 格式](https://git-scm.com/docs/pack-format) + +## 3. 能力协商及公共请求上下文 + +扩展现有 GET /api/v1/snapshots/capabilities 的传输 envelope： + +~~~json +{ + "transfer": { + "protocol_version": 1, + "metadata": true, + "small_packs": true, + "cached_packs": false, + "chunk_reads": false, + "pack_encodings": ["tar", "tar.zstd"], + "limits": { + "directory_entries": 256, + "metadata_bytes": 1048576, + "lookup_paths": 128, + "small_blob_bytes": 262144, + "pack_objects": 128, + "pack_raw_bytes": 4194304, + "pack_tar_bytes": 5242880, + "pack_wire_bytes": 6291456, + "chunk_bytes": 1048576, + "chunk_page_entries": 256 + } + } +} +~~~ + +这是启用后的格式示例；当前不得返回 true。能力取客户端支持与服务端声明的交集，客户端还可施加更低资源限制。metadata、small_packs、cached_packs、chunk_reads 分别经过验收再开启，不宣称一个开关就完成全部功能。 + +挂载首先通过既有 resolve 原子取得具体 view_id 和租约。之后所有新路由均以 /api/v1/snapshots/{view_id} 为前缀，拒绝以 latest 作为文件读取 ID。 + +公共 header： + +~~~http +Authorization: Bearer +X-Mega-Snapshot-Lease: +~~~ + +租约负责保留，token 负责当前授权。请求在同一服务 origin 内完成，不重定向携带凭据的请求。路径出现在请求体或 query 时进行规范校验、正确百分号编码，禁止将用户路径用于拼接后端主机地址。敏感 header 与路径不进入默认访问日志。 + +路径解析由固定 view 的 native/bindings 完成。缓存命中、304、包缓存、分块缓存都在当前授权检查之后返回。只知道 OID、pack ID 或 chunk digest 不能获取内容。 + +## 4. 目录和 lookup：正文之前先获取元数据 + +沿用 GET /{view_id}/tree?path=...&limit=256&cursor=...。这里及后文的 /{view_id} 均省略公共 /api/v1/snapshots 前缀。 + +目录页示例： + +~~~json +{ + "view_id": "V100", + "path": "/project/app/src", + "entries": [ + { + "name": "main.rs", + "kind": "file", + "mode": "100644", + "blob_oid": "A", + "raw_size": 8192, + "source_id": "MAIN" + }, + { + "name": "parser.rs", + "kind": "file", + "mode": "100644", + "blob_oid": "B", + "raw_size": 8192, + "source_id": "MAIN" + } + ], + "next_cursor": null, + "pack_hints": [] +} +~~~ + +- file/executable/symlink 必须有准确 raw_size；symlink 是目标文本长度。Git tree 不包含文件长度，Mega 从已验证的对象长度元数据批量读取，不能对每个文件发起远端 HEAD 或下载正文。缺元数据返回可重试 METADATA_NOT_READY，不能填 0。 +- directory 使用单独的结构：name/kind/mode；真实 source tree 可带 tree_oid，聚合目录不伪造 Git tree OID。目录 FUSE 属性采用固定客户端规则，不把子项数量当文件字节数。 +- gitlink 明确返回 kind=gitlink 和 commit_oid；不伪装为可自动遍历的子目录。 +- cursor 绑定 view、规范目录路径、排序规则、最后扫描 key、查询参数及授权政策代次并防篡改。按 UTF-8 名称字节排序；服务端以索引 seek 合并固定 native/binding 子项，不能每页扫描整个 registry 或巨型 Git tree。 +- 每页同时限制返回数、响应字节数和扫描工作量。ACL 过滤后可以出现空 entries + 非空 cursor；只有 next_cursor=null 才表示枚举完成。政策变化使旧 cursor 返回 CURSOR_STALE，从同一 view 重新枚举。 +- lookup 的缺失只能由固定路径解析确认；网络错误或未加载完的一页不能产生 ENOENT。directory handle 与 cookie 固定 view 和本次枚举状态。 +- 大目录的 source tree 可在接收或后台验证后建立可 seek 的派生条目索引，版本映射指向同一 tree；索引未就绪明确报错，不阻塞在线请求做无限全扫描。 + +新增 POST /{view_id}/transfer/lookup，body 为 {"paths":[...]}，最多 128 个绝对路径。单次请求在服务端沿固定树完成深路径解析，返回逐路径元数据或明确错误；共享父路径批量读取。这将深路径的网络往返收敛为一次，不宣称服务端只做一次树访问。 + +目录列表是受认证 Mega 给出的投影与长度信息，不是“一个目录页可独立通过 Git tree 哈希证明”。原始 tree API 保留完整 Git 哈希校验；如果将来要求对聚合分页进行独立密码学证明，需要另行提供完整证明链。 + +## 5. 按缺失列表打包 + +POST /{view_id}/transfer/pack 是只读、可安全重试的批量下载操作： + +~~~json +{ + "encoding": "tar.zstd", + "items": [ + { "path": "/project/app/src/main.rs", "expected_blob_oid": "A" }, + { "path": "/project/app/src/parser.rs", "expected_blob_oid": "B" } + ] +} +~~~ + +规则： + +1. 以 items 而非“客户端没有哪些全库对象”作为输入；最多 128 项、128 KiB 请求体。不发送全局 have 列表或 Bloom filter。 +2. 服务端在固定 view 中解析每一项，验证当前权限、对象类型和 expected_blob_oid。输入路径必须在同一固定 SourceSnapshot/scope 内；跨 import 边界拆包，错误码 MIXED_SOURCE_BATCH。 +3. 输入允许多个合法路径引用同一 blob，但授权逐路径检查，运输按 (kind, object_format, OID) 去重；总大小按唯一对象计算。 +4. 任一项权限、版本、大小或对象状态不符合要求，整个请求在发送 200 前失败。返回非 200 JSON 错误；不把失败对象从成功清单中静默删掉。需要缩小请求时客户端拆分。 +5. 每对象不超过 256 KiB，总 blob payload 不超过 4 MiB。使用已验证长度预检，并在读取中重新限制；未知大小不能通过无限读取“探测”。 +6. 服务端先在受限内存或临时文件中构造并验证完整小包，得到 wire 长度与 pack ID，再开始响应。这有有限的打包等待，必须记录 time-to-first-byte；不能伪称零准备延迟。相同对象集合和编码请求可以合并构建，但每个调用方分别鉴权。 +7. 每对象服务器读取时验证实际 Git 哈希；已通过受信任不可变存储验证的缓存可复用。客户端仍验证全部收到的对象。 +8. POST 表示有请求体的读取，不创建发布操作，不推进 latest；代理不自动缓存 POST。复用由服务端包缓存管理。 + +响应： + +~~~http +HTTP/2 200 +Content-Type: application/zstd +X-Mega-Archive-Format: tar-v1 +X-Mega-Pack-Id: sha256: +Content-Length: +Cache-Control: private, no-store, no-transform +~~~ + +tar.zstd 是实际表示，HTTP Content-Encoding 不再设置 zstd，避免客户端或代理双重解压。客户端通过响应 type 和 archive-format 选择解码器。协商 encoding=tar 时返回 application/x-tar，同样限制 tar/wire 字节数。 + +小包为完成度边界；HTTP 200 仅表示响应开始，不等于包已验证完成。中途超时、长度不符、缺成员、压缩结尾异常都使本次包失败。 + +## 6. 小包格式和校验 + +解压后的 ustar 内容依次为 manifest.json、按 OID 字节序排列的唯一 blob 成员、两个 512-byte 零结束块。拒绝结束块之后的非零内容；可接受的零填充仍计入 tar 上限。 + +~~~json +{ + "schema_version": 1, + "object_format": "sha1", + "objects": [ + { "oid": "A", "kind": "blob", "raw_size": 8192 }, + { "oid": "B", "kind": "blob", "raw_size": 8192 } + ] +} +~~~ + +manifest 最多 64 KiB，只包含上述字段；blob 文件名严格为 objects/blob/<40位小写十六进制OID>。manifest 与对象成员均为普通 tar 文件，uid/gid/mtime=0、mode=0644、空 uname/gname，使用标准 ustar header 和八进制 size；不允许 PAX 扩展、链接、设备、目录、稀疏文件或重复成员。 + +tar 模式与 Git 可执行位没有对应关系。Git symlink blob 作为普通对象字节运输；从包中不会创建符号链接。 + +客户端： + +1. 限制 wire、zstd window、解压 tar、单成员和总 payload 字节数；限额在分配/写入前检查。一个 zstd frame，禁止跨包 dictionary、skippable frame 和拼接 frame；window 最大 8 MiB。 +2. 先验证 manifest 中的对象集合与请求预期完全一致，再读取各成员。tar header 长度必须等于 manifest.raw_size。 +3. 将成员流入受控临时缓存，同时计算 SHA1("blob " + decimal_length + NUL + payload)，与预期 OID 比较。任何类似 Git 头的文件前缀都属于文件内容。 +4. 不调用“按 tar 路径解包到 workspace”的通用操作。只用已校验 OID 生成缓存位置；写入完成后原子替换到本地对象缓存。 +5. wire bytes 计算 SHA-256 并对比 pack ID；检查 HTTP 完整结束、压缩结束、tar 结束与成员覆盖。包 ID 是压缩表示 ID，不是新的文件或 monorepo 版本。 +6. 断流前已完整通过 Git 哈希验证的对象可以保留；未完成成员删除。重试重新列出缺失对象，形成更小的包。不能使用压缩字节 offset 恢复半个解压流。 + +采用标准 tar 的结构，但客户端采用比通用解包工具更严格的结束及类型检查。[tar 格式](https://www.gnu.org/software/tar/manual/html_section/Standard.html)； +zstd frame 和 window 规则参考 [RFC 8878](https://www.rfc-editor.org/rfc/rfc8878.html) 与 [RFC 9659](https://www.rfc-editor.org/rfc/rfc9659.html)。 + +## 7. 热点目录包：复用、选择与失效 + +目录页可以返回有界 pack_hints。示例中成员路径相对当前目录： + +~~~json +{ + "pack_id": "P", + "encoding": "tar.zstd", + "wire_bytes": 614400, + "raw_bytes": 1048576, + "members": [ + { "name": "main.rs", "blob_oid": "A", "raw_size": 8192 }, + { "name": "parser.rs", "blob_oid": "B", "raw_size": 8192 } + ] +} +~~~ + +这是节选示意；真实 members 必须列全并与 raw_bytes 相符。hint 可省略，最多覆盖当前返回页的 128 个直接子文件，和 entries 一起受 1 MiB 响应限制。页布局改变允许不给 hint，不影响目录正确性。 + +GET /{view_id}/transfer/packages/{pack_id}?path= 读取已存在的小包。Mega 必须验证该包的全部成员在请求目录和固定 source 下对应相同 OID，并检查当前每个成员权限、租约。内部包缓存命中不能跳过这个步骤。 + +热点包按直接父目录、source/scope、稳定名称排序分组，目标 1 MiB、上限同按需包。不跨 source，不无界递归收集子目录；超大文件排除。只有受请求触发或有预算的热点预热才构建。不同 view 中相同对象集合和编码可复用包字节，目录到包的关联另存并按版本验证。 + +插入文件可能改变相邻分组，不承诺包边界稳定。客户端缓存以 blob 为单位，因此分组变化不会使已有 blob 失效。 + +ScorpioFS 的静态基线选择规则（另与研究设计中的成本选择器对照）： + +- 当前明确需要/已有访问记录预计将用到的文件中，包内至少 75% 原始字节尚未缓存，且浪费预算允许时，才选现成包。 +- 已缓存大多数文件时，POST 精确缺失列表；单次 GET 某个文件没有被强制转换为整包下载。 +- 不将“readdir 看到了一个文件”当成“马上会读取文件”。无历史线索时由并发需求聚合；目录预取可关闭。 +- 单文件阻塞读取立即发出，聚合窗口最多 1 ms，不等待包凑满；已经有并发请求的缺失优先组成包。 +- 预取最多 2 个并发包、8 MiB 原始内容；高优先级读取、内存/磁盘压力或低命中率出现时暂停。普通读取不等待预取队列排空。 + +GET 包被缓存淘汰后返回 404 PACK_EVICTED，客户端用同一版本的精确对象列表重建；这不代表文件不存在。包 ID 丢失不能自动改读最新版本。 + +## 8. 单文件与大文件 + +新增 GET /{view_id}/transfer/blob?path=...&expected_blob_oid=... 提供不超过 1 MiB 的完整对象，返回原始 application/octet-stream，HTTP 内容编码保持 identity。其 Git 哈希验证同小包；空 blob 合法。256 KiB～1 MiB 文件走此接口，超过 1 MiB 默认走分块。 + +原先 source raw-object API 仍按其现有上限和完整哈希规则工作，用于兼容与独立校验；新客户端不得因 chunk endpoint 失败就无限制退回整文件下载。 + +大文件先请求 GET /{view_id}/transfer/chunk-map?path=...&expected_blob_oid=...&cursor=...： + +~~~json +{ + "view_id": "V100", + "blob_oid": "COMPILER", + "map_id": "MAP", + "raw_size": 3221225472, + "chunk_size": 1048576, + "chunk_count": 3072, + "chunks": [ + { + "index": 100, + "offset": 104857600, + "raw_size": 1048576, + "sha256": "C100" + } + ], + "next_cursor": "NEXT" +} +~~~ + +分页示意：首请求可带 start_chunk=100；后续只能用 cursor，cursor 绑定 view、blob、map、分页范围和授权上下文。每页最多 256 项、128 KiB，不能为了读取一块而下载整个 TB 文件的清单。 + +块信息结构： + +~~~text +BlobTransferInfo { + blob_oid, object_format, raw_size, + chunk_size, chunk_count, map_id, + state: preparing | ready | corrupt +} + +ChunkEntry { + index, raw_size, sha256 +} +~~~ + +offset = index * chunk_size，不另存一个可矛盾的偏移。除最后一块之外长度固定；chunk_count = ceil(raw_size/chunk_size)。分块只面向大于 1 MiB 的 blob，没有“空块”的例外。 + +map_id 定义为 SHA-256： +ASCII "mega.blob-chunks.v1" + NUL， +再依次拼接 object_format 的单字节值 1（sha1）、20-byte blob OID、 +u64-BE raw_size、u32-BE chunk_size、u64-BE chunk_count， +最后按 index 顺序拼接每项的 u32-BE raw_size 和 32-byte SHA-256。 +ID 显示为 sha256: + 小写十六进制。 + +这项编码只用于派生分块表身份。客户端读取一页不能独立计算完整 map_id，也不能从 Git OID 推导分块哈希；它信任经认证 Mega 对分页内容、map_id 和 blob 的绑定。已获用户确认。全表被获取时可额外重算 ID，但这不是每次随机读取的前提。 + +读取块： + +~~~http +GET /api/v1/snapshots/V100/transfer/chunks/MAP/100?path=/toolchains/compiler.bin&expected_blob_oid=COMPILER +Authorization: Bearer +X-Mega-Snapshot-Lease: +Accept-Encoding: identity +~~~ + +200 返回完整块的原字节，Content-Length 必须精确匹配已取得条目。客户端计算 SHA-256，对照分块表验证后才把所需范围交给 FUSE。 + +一块是一个 HTTP resource；这里不使用文件 Range，也不把整文件 zstd 流切片。HTTP Range 对内容编码后的表示计偏移，和原文件 offset 容易混淆。[RFC 9110 §14.1.2](https://www.rfc-editor.org/rfc/rfc9110.html#section-14.1.2) + +块完整性只在完整块验证后成立；断线重取这一块。若下载齐全部块，客户端可流式重算 Git blob 哈希后把它提升为完整 blob 缓存；只有部分块时保持“按可信分块表验证”状态。 + +## 9. 服务端存储与任务调度 + +需要的逻辑记录，不代表本次已新增数据库表： + +~~~text +ObjectLength(domain, kind, algorithm, oid) -> verified_raw_size +DirectoryEntries(source_tree_oid, name) -> kind, oid +PackCache(domain, member_set, encoding_revision) -> pack_id, bytes_location, sizes +DirectoryPack(view/source-tree, directory, group) -> pack_id, member_paths +BlobTransferInfo(domain, blob_oid, chunk_size) -> ready map_id +ChunkTable(map_id, index) -> raw_size, sha256, storage_location +~~~ + +domain 是服务端授权隔离域，不能由客户端自己指定。文件身份与权限关联分别保存；同一物理对象去重不等于允许跨域读取。 + +- 新 Git 对象接收/合并路径应在已验证字节流上记录准确长度。存量对象受限回填；GET tree 不做同步无限回填。 +- source/binding 路径解析合并公共前缀，按 source 和唯一 OID 批量查 size。禁止 N 次单对象查询替代一个真正的批量查询。 +- 包构建按对象集合合并重复任务；服务实例维护 CPU、数据库、内存和临时磁盘信号量，多副本共享缓存但各请求仍鉴权。初始建议每实例 4 个 pack builder，最终依据 CPU 和存储能力配置。 +- zstd 默认 level 1；预热可评估 level 3，已有压缩对象采用 tar 或原始单文件；不在请求线程进行无界高等级压缩。 +- 大文件的分块任务必须读取重建后的 blob、验证完整 Git OID，再将块和分块表原子标为 ready。验证失败标为 corrupt，临时块不能对外可见。 +- 大文件支持范围读取的原始存储可配合分块索引；Git pack/delta 先受限还原到磁盘或块对象存储。不能让每块读取重新产生 O(file_size) 解码工作。 +- 对存量大 blob 建表会有一次 O(file_size) 成本。preparing 返回 503 CHUNK_MAP_NOT_READY + Retry-After，排队去重；新导入可在 ingest 时准备。没有完整图时不承诺“首次冷读取只需一块的服务端 IO”。 +- 分块页和块存储一旦对客户端发布，其可用性要跟随 view/source 租约和请求 in-flight pin。回收与续租协调，不得把有效读依赖作为普通 LRU 删除。小包是可重建性能缓存，可提前淘汰。 +- 原始 Git pack/base 保留仍由既有 GC 契约负责。分块缓存不是完成 Git 对象保留审计的替代品。 + +首次试点显式配置总缓存字节限额和临时磁盘限额，磁盘不足停止预热/构建并返回容量错误。实现必须有流式对象读取接口；不得用 Vec 容纳 GB 级文件后声称支持分块。 + +## 10. ScorpioFS 缓存与 FUSE 接入 + +~~~text +PathCache(domain, view_id, path) -> source, kind, mode, blob_oid, size +BlobCache(domain, algorithm, kind, oid) -> verified whole payload +ChunkCache(domain, sha256, raw_size) -> verified chunk payload +ChunkMapCache(domain, blob_oid, map_id, page) -> authenticated entries +~~~ + +路径缓存绑定版本；对象内容不强制绑定 view，以复用相同文件。内容命中前仍需有效的路径/版本和授权上下文。chunk 命中必须先取得对应 blob 的受信任 map entry，不能由裸 SHA-256 值直接读缓存。 + +具体接入： + +1. 增加目录元数据客户端和请求调度层，让 lookup/getattr/readdir 使用准确大小；新的 lower 不采用旧的“未知大小暂填 0”行为。 +2. 同一 (domain, kind, OID) 的并发缺失合并，取消一个 waiter 不取消其他仍需要的 waiter。 +3. 请求分为 demand 和 prefetch，保留至少一个并发槽处理元数据/阻塞读取；不能仅依赖 HTTP/2 priority header。 +4. 初始上限：8 个数据流、2 个元数据流、64 MiB 用户态传输缓冲，单包以流式临时文件解包。这些不包含内核 page cache 与持久化缓存，监控中必须分别报告。 +5. 大文件新增 range-reader 接口供 FUSE read(offset,length) 使用，不让现有 read_file -> Bytes API 返回“其实只有一块”的伪完整文件。 +6. 块覆盖公式为 floor(offset/chunk_size) 到 floor((min(offset+length,file_size)-1)/chunk_size)。checked arithmetic；length=0 或 offset>=EOF 返回空，越 EOF 裁剪。跨块依次拼接已验证部分，不用零填充尚未下载的洞。 +7. 顺序读取可预取后续 2 块；随机访问降低预取，负载压力下停用。所有预取都固定相同 view。 +8. 本地内容对象原子写入，进程重启清理不完整临时文件。FUSE 句柄携带既有版本与代次；旧响应不能填入新版本的路径/属性缓存。 + +缓存和 kernel page cache 无法收回已经交给应用的数据。服务端对新请求逐次检查当前授权；在线客户端定期确认授权，在拒绝后停止新读取并使可失效元数据失效。任何短期授权缓存有效期都需由服务端明确给出，缺省不授予离线权限；不宣称能瞬间撤销已读字节或 mmap 页面。 + +## 11. 重试、失败与降级 + +| 情况 | 服务端/客户端行为 | +| --- | --- | +| 401/403 | 当前请求失败，停止有关后台预取；不重试到其他 origin | +| 410 SNAPSHOT_EXPIRED | 停止相关版本读取，要求显式恢复租约或重新建立 workspace | +| 404 PATH_NOT_FOUND | 仅固定目录解析确认时产生 ENOENT | +| 404 PACK_EVICTED | 同一版本重新按缺失列表请求包 | +| 409 EXPECTED_OBJECT_MISMATCH | 目录与请求不一致，失败并检查固定版本；不自动接受新 OID | +| 409 CURSOR_STALE | 同一版本重开枚举，不能沿用旧 cookie | +| 413 BATCH_LIMIT_EXCEEDED | 按服务器上限拆分，不把一个大文件无限重试为小包 | +| 429 / 503 | 遵循 Retry-After 与有抖动退避，先暂停预取 | +| 503 METADATA_NOT_READY / CHUNK_MAP_NOT_READY | 有界等待后台准备，不报空文件，不悄悄整库预取 | +| 503 OBJECT_UNAVAILABLE / CORRUPT | 暴露 I/O 错误，不报告“路径不存在” | +| 压缩/哈希/长度错误 | 丢弃未验证数据；有限重试后 I/O 错误并诊断，禁止写入内容缓存 | + +初始网络重试预算：单次逻辑读最多 3 次、总计 30 秒；后端准备任务可由调用方显式等待更久，不由内核读无限挂起。客户端取消请求时服务端停止无其他消费者的下载或任务；完整缓存包可供其他请求继续使用。 + +small_packs 不支持时，可以退回固定 source 的完整对象接口，但保持字节上限、授权与租约。cached_packs 不支持时使用精确列表包。chunk_reads 不支持且对象超限时明确失败，不能切换到 legacy latest。 + +## 12. 交付顺序及验收 + +| 切片 | Mega | ScorpioFS | 验收 | +| --- | --- | --- | --- | +| T1 | 目录准确长度、分页、固定路径 lookup | 元数据缓存、stat、逐对象基线 | stat 不下载正文，深路径一次 lookup RPC | +| T2 | 精确小包接口、标准编码、资源限制 | 合并缺失、校验解包、内容缓存 | 冷构建小文件包传输与逐对象基线对比 | +| T3 | 热点包缓存及 hints | 按需选择、限额预取、跨版本复用 | 更新 10/1000 文件只传缺失内容 | +| T4 | 大文件 map、随机读存储、保留 | range reader、块缓存与重试 | 3 GiB 文件局部读、故障与双版本 FUSE | + +实际路由进入 mono，调度/路径/包服务进入 Ceres，持久化和流式对象读取进入 Jupiter/对象存储。ScorpioFS 在 snapshot 下增加协议客户端、对象缓存与调度器，由 Dicfuse lower 调用；Libra 不接管这些传输。 + +必须覆盖： + +- Native + import A/B 两版本同时读，旧版本首次惰性请求不查询新 refs。 +- 独立 Git oracle 验证解包对象，binary/空文件/可执行文件/symlink blob，重复 OID、Unicode 路径和恶意 tar header。 +- tar 每个阶段断流、manifest 多/缺成员、超限、zstd window/字典/多帧异常、压缩及 Git 哈希不符。 +- 同一缓存包在另一个 source、路径复用、权限撤销后的访问；包包含一个无权限文件时不能发送任何正文。 +- 超大目录的分页字节/扫描上限、ACL 空页、跨版本 cursor、无效页不能产生假 ENOENT。 +- 多个请求同时读同一 OID、部分取消、磁盘满、包构建失败、冷并发请求不会重复压缩 N 次。 +- 分块 offset/EOF/跨块/超大整数、map 与 blob 不匹配、错误块摘要、部分下载不能提升完整 Git blob 状态。 +- ingest 未完成、map 未就绪、租约与 GC 竞争、pack delta 还原；分别测试新导入与存量冷数据。 + +性能实验需使用同一固定版本、文件集合、服务端 CPU/IO 预算和客户端并发： + +| 工作负载 | 核心指标 | +| --- | --- | +| 1,000 × 8 KiB 同 source/目录文件，冷缓存 | 需求已知且按 128 个聚合时为 8 次包请求，目录分页另计；纯按需另报实际数量，对比同并发单对象 GET | +| 上述文件只改 10 个，热缓存 | 理想受控场景新增正文为 80 KiB，允许 tar/压缩开销；不能下载其余 990 个正文 | +| 深 10 层单路径读取 | 元数据一次 lookup RPC，记录服务端树读取而非隐藏 IO | +| 百万目录项、少量子目录工作集 | 页工作量、数据库查询、RSS 不随全库大小线性增长 | +| 3 GiB 文件的 100 MiB 偏移读取 64 KiB | map ready 且关预取时一块 1 MiB 正文；map/HTTP overhead 另计 | +| RTT 1/20/80 ms，带宽 100 Mbit/s / 1 Gbit/s | 冷/热构建时长、p50/p95 首文件时间、请求数、wire/raw 字节、CPU | +| 并发构建与随机读取 | 队列等待、pack 命中、预取命中与浪费、峰值内存/磁盘、解码工作量 | + +预取关闭/开启、tar/ tar.zstd、逐对象 HTTP/2 均作为基线。Git partial clone 也记录了逐对象获取的代价及批量预取的做法，但这不是本系统的加速数据。[Git partial clone](https://git-scm.com/docs/partial-clone) + +本次完成的是协议设计。上述基准尚未运行；小包大小、并发和预取阈值必须通过 T2/T3 实测后调整，不能把算出来的请求数量当成已实现的构建性能。 diff --git a/docs/spec/source-snapshot-v1.md b/docs/spec/source-snapshot-v1.md new file mode 100644 index 000000000..18106a44f --- /dev/null +++ b/docs/spec/source-snapshot-v1.md @@ -0,0 +1,84 @@ +# Source snapshot v1 contract + +Status: implemented identity/read foundations, 2026-09-06. This contract does not advertise a deployed snapshot capability. Namespace publication, leases, scope attestation coverage and FUSE integration remain separate gates in the versioning specs. + +## JSON and validation + +The source descriptor has exactly these five fields: + +```json +{ + "source_id": "11111111-1111-4111-8111-111111111111", + "scope_path": "/project/a", + "object_format": "sha1", + "commit_oid": "1111111111111111111111111111111111111111", + "root_tree_oid": "4b825dc642cb6eb9a060e54bf8d69288fbee4904" +} + +``` + +The hashes above illustrate structure, not a deployed source or a claimed commit/tree relationship. + +- source_id is a non-nil, lowercase, hyphenated UUID persisted by Mega. Its server-side mapping includes instance, backend kind and repo ID. Paths are not source IDs; recreating a different logical source must not reuse an ID. +- scope_path is canonical absolute UTF-8. / is the root; other paths cannot end in / or contain empty, dot or parent components or NUL. Limit: 4096 UTF-8 bytes per protocol path, 255 per component. This protocol limit is not a guarantee that any host-local mountpoint prefix fits an OS path limit. +- Names retain case, Unicode composition, plus signs and literal backslashes. No Windows path normalization is applied. Non-UTF-8 names are unsupported in v1. +- object_format is sha1. Object IDs contain exactly 40 lowercase hexadecimal digits. Future algorithms require negotiation, not automatic acceptance. +- Unknown fields, invalid IDs and unknown algorithm tags fail deserialization. Structural validation does not prove scope, object membership or commit.tree: Mega must attest those relationships. + +The source_ref selector requires a fully qualified refs/heads/... or refs/tags/... name. The source_commit selector accepts only a commit OID, never an arbitrary tree/tag OID. The compatibility parser in existing Mega browsing APIs still accepts an unqualified tag name; the new typed contract does not. + +## Canonical source identity + +Do not hash a JSON serialization. Canonical source bytes consist of: + +1. The ASCII domain mega.source-snapshot.v1 followed by one NUL byte. +2. The five fields in this order: source_id, scope_path, object_format, commit_oid, root_tree_oid. +3. Each field is encoded as its unsigned 32-bit big-endian UTF-8 byte length followed immediately by those UTF-8 bytes. OIDs are lowercase hex text here, not raw 20-byte values. + +source identity = sha256: followed by the lowercase SHA-256 hex digest of those bytes. + +The same shared fixture is tested by both implementations. It includes an ASCII scope and a Unicode/plus-sign scope. The first vector's identity is sha256:6e3f8a7e41d3a9759bc05cbc1dab153ad27ba0e0ff494f7692392dbfd5a95451. Fixture bytes/digests were independently computed with .NET; Ceres uses RustCrypto SHA-256 and ScorpioFS uses ring. + +This identity includes commit provenance. It is not namespace view_id, publication_seq, a lease, or a projection_key. A same-tree/different-commit pair has different source identities but may still share verified physical objects within an authorized domain. + +## Immutable object boundary + +ScorpioFS SourceReader owns a fixed SourceSnapshot and only asks ObjectBackend for (source, object kind, OID, root-relative source_path, byte limit). It exposes no mutable ref selector. A different version requires another reader; a caller cannot mutate the descriptor held by an existing reader. + +source_path is a membership/authorization context, not a lookup through current routing: the server must walk from the descriptor's fixed root and verify the resulting kind/OID before returning an object. Root uses the empty relative path. This avoids a whole-repository reachability scan per request. Signed object tickets may optimize the same check later; arbitrary caller-supplied OIDs are never sufficient proof. + +Backends must check current authorization and retention even on a global CAS hit, enforce limits during download, and return raw object payloads. The client verifies SHA-1 over Git's type + space + decimal length + NUL + payload. A file beginning with Git-like header bytes retains those bytes. + +Tree traversal is relative to the source root. Prefix neighbors such as /project/ab do not match scope /project/a. A scope commit's tree is already rooted at the scope; the prefix is never applied twice. Tree names, entry modes, symlink targets, missing paths and failed object fetches remain distinct. + +The initial reader is a bounded whole-object implementation: default limits are 16 MiB/tree and 64 MiB/blob. It returns an explicit size-limit error, never empty bytes, on oversized objects. This is not the final streaming/CAS/FUSE adapter or a claim that stat is metadata-only. Namespace routing, chunked large-object reads and controlled workspace generation changes are not completed by these tests. + +## Mega source observations and scope proofs + +SourceCatalog registers stable backend IDs and resolves typed selectors. A new import observation uses its registered root and a repo-scoped commit/tag resolution. A new native observation requires an exact scoped ref whose stored root agrees with commit.tree; it records native_ref_observed, not a claim that an older writer emitted a creation proof. Native projection derives a child by walking an already attested fixed root and preserves the base commit provenance. + +Explicit native commits without a proof for the requested scope return SCOPE_UNKNOWN. A recorded descriptor can be resolved after refs or registry entries are removed; a reused path assigned to a different repo ID receives a different source ID. No current registry lookup is used to read an already attested source. + +The catalog checks descriptor attestation and walks root-relative paths to bind object kind/OID to source membership. It strictly decodes UTF-8 trees and checks SHA-1 independent of git-internal's thread-local algorithm. It is an internal metadata service, not an authorization or retention grant. Public HTTP reads must add those checks, and no snapshot endpoint or capability is enabled by the catalog alone. Observing individual sources is not an atomic multi-source namespace publication. Existing commit metadata is trusted ingestion state; this foundation does not claim raw commit/tag payload re-verification or complete proof capture by every writer. + +## Source object HTTP binding (client adapter implemented) + +GET api/v1/sources/{source_id}/trees/{oid} or blobs/{oid}, relative to the configured server base URL, carries exactly five percent-encoded query fields: object_format, scope_path, commit_oid, root_tree_oid and source_path. The path supplies source_id and the expected object kind/OID; together these reconstruct the attested descriptor and fixed-root membership request. There is no ref/latest query. + +Authorization: Bearer carries a current Mono access token; X-Mega-Snapshot-Lease carries the retention lease identifier. Neither is a query parameter or Debug field. The server must validate both and must not let the lease substitute for source/scope authorization. HTTP access logging should redact these headers and avoid logging private query paths. + +A successful full object response is HTTP 200, Content-Type application/octet-stream, with raw bytes. The client does not follow redirects, accept partial/204 responses as full objects, or treat JSON/HTML login/error pages as objects. SourceReader verifies the returned Git hash. The adapter checks Content-Length and also bounds collection of streamed chunks when length is absent. 401/403 become Forbidden, 410 becomes Expired; other failures, including object/source 404, remain Unavailable rather than being misreported as an absent directory entry. A missing entry discovered in a verified tree is a separate PathNotFound result. + +The adapter requires HTTPS except for loopback HTTP test/development servers, rejects base URLs containing userinfo/query/fragment, sets connect/request deadlines, and retains no reqwest URL-bearing error text. It does not acquire or renew leases, negotiate capabilities, authorize requests on behalf of Mega, or connect existing mounts automatically. Local Axum transport tests are not a deployed Mega end-to-end test. The server routes remain an implementation gate under the confirmed default-off policy. + +## Verification + +Run the relevant repository command: + +```sh +cargo test -p ceres --lib snapshot --locked +cargo test --lib snapshot --locked + +``` + +The ScorpioFS reader fixtures obtain their OIDs from git hash-object --stdin (without -w). Git must be installed, but these tests need no network, FUSE mount, mutable global Git configuration or existing repository objects. diff --git a/jupiter-migrate/Cargo.toml b/jupiter-migrate/Cargo.toml index 30a1a0858..e103f2989 100644 --- a/jupiter-migrate/Cargo.toml +++ b/jupiter-migrate/Cargo.toml @@ -22,5 +22,6 @@ tracing = { workspace = true } chrono = { workspace = true } [dev-dependencies] +url = { workspace = true } tempfile = { workspace = true } tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/jupiter-migrate/README.md b/jupiter-migrate/README.md index 1d74eaed0..59da45384 100644 --- a/jupiter-migrate/README.md +++ b/jupiter-migrate/README.md @@ -51,7 +51,88 @@ Review generated diffs in `jupiter/callisto/src/` before committing. Join call sites that need those relations use `callisto::entity_ext::::Relation`, not the generated entity `Relation`. -## Library API +## Snapshot entity generation and database gates + +The snapshot source identity migration is additive; it does not create a published +namespace or guess historical scope mappings. To reproduce only its entities, +first create a new temporary directory, then run the following with its absolute +path substituted for ``: + +```bash +cargo run -p jupiter-migrate --example snapshot_schema -- /schema.db +sea-orm-cli generate entity -u sqlite:///schema.db -o /entities --tables snapshot_instance,snapshot_source,source_commit_scope,namespace_node --with-serde both --entity-format dense +``` + +The example rejects existing database files. Review and copy only those four +generated entity files into Callisto; merge their module/prelude registrations +without replacing the existing registries or `entity_ext`. The initial generation +uses sea-orm-cli 2.0.2. SQLite generation verifies the actual migration schema; +PostgreSQL runtime/transaction tests separately check backend compatibility. The +namespace node entity uses `i64`/SQL BIGINT and a timezone-aware timestamp on both +backends. Its payload has a 16 KiB database check as well as storage-layer checks. + +`source_commit_scope` indexes a SHA-256 scope key and retains the full UTF-8 path +as data. This avoids placing multi-kilobyte paths in a PostgreSQL btree key. The +storage facade checks the full path on reads and rejects conflicting immutable +attestations. There is no cascading FK from mutable refs or repo paths to proof records. + +The forward migration `m20260906_145000_snapshot_utc_timestamps` converts the three +earlier source tables' PostgreSQL `created_at` columns from TIMESTAMP to +TIMESTAMPTZ, matching their generated `DateTimeUtc` models. It explicitly treats +legacy values as UTC and is a no-op on SQLite. This is a compatibility repair, +not a namespace backfill. **Before applying to an already-used draft database, +verify that those legacy values were written in UTC.** A legacy deployment using +a non-UTC connection timezone may have stored local wall times; audit/repair +those values before this migration. Do not infer that the migration can discover +their original timezone. Production application rollback must retain these +historical tables; `down` is exercised only on disposable test schemas. + +To run the PostgreSQL gates, point `MEGA_SNAPSHOT_TEST_DATABASE_URL` at an explicit +loopback-only disposable database named `snapshot_test`, never a production DB: + +```bash +cargo test -p jupiter-migrate --lib snapshot --locked -- --include-ignored --nocapture +cargo test -p jupiter --lib snapshot_storage --locked +cargo test -p jupiter --lib namespace_storage --locked +cargo test -p ceres --lib snapshot --locked -- --include-ignored --nocapture +``` + +The ignored PostgreSQL tests fail without the URL, require loopback and the exact +test database name, and create independent schemas rather than refreshing public +tables. They retain those schemas for diagnosis. Verified locally on PostgreSQL +16.15: fresh migrations, concurrent source/node insertion, long scope paths, +transaction rollback, reconnect/readback and immutable radix roots. A separate +upgrade test verifies known UTC legacy values through forward/down/up migration +under an America/Los_Angeles session. The focused CI job also runs the ignored +million-binding index test. These gates do not validate publication, leases, GC, +writer fencing or the entire workspace; those remain separate acceptance work. + +## Publication metadata generation and tests + +The additive `m20260906_160000_namespace_publication` migration creates five more +tables without initializing a head or enabling an API. Generate only its models +from a new disposable schema (again use a fresh `` absolute directory): + +```bash +cargo run -p jupiter-migrate --example snapshot_schema -- /publication.db +sea-orm-cli generate entity -u sqlite:///publication.db -o /entities --tables namespace_view,namespace_head,namespace_publication,snapshot_operation,namespace_outbox --with-serde both --entity-format dense +``` + +Copy the five generated models, merge registries and preserve `entity_ext` as +above. All counters use BIGINT and timestamps are timezone-aware. The snapshot +migration roundtrip test now checks all nine snapshot tables. The UTC upgrade +regression locates the UTC migration by name rather than assuming it is last. + +`cargo test -p jupiter --lib publication_storage --locked -- --include-ignored +--nocapture` explicitly runs both SQLite and disposable PostgreSQL tests using +the same guarded URL. They cover the real `import_refs` table participating in +the publication transaction, operation replay, CAS, rollback after head update, +concurrent duplicate/expected-old requests and reconnect receipt lookup. See +[publication core](../docs/spec/namespace-publication-core.md) for the precise +evidence boundary: all production writers, composition, pins, authorization and +outbox delivery remain separate requirements. + +## Library API reference ```rust use jupiter_migrate::{apply_migrations, Migrator}; diff --git a/jupiter-migrate/examples/snapshot_schema.rs b/jupiter-migrate/examples/snapshot_schema.rs new file mode 100644 index 000000000..e12f255b5 --- /dev/null +++ b/jupiter-migrate/examples/snapshot_schema.rs @@ -0,0 +1,32 @@ +//! Prepare a NEW, local SQLite database from the real migrations for codegen. +//! Usage: cargo run -p jupiter-migrate --example snapshot_schema -- /tmp/new.db +//! Existing files are rejected so a developer cannot overwrite a database. + +use std::{fs::OpenOptions, path::PathBuf}; + +use sea_orm::{ConnectOptions, Database}; + +#[tokio::main] +async fn main() -> Result<(), Box> { + let path = PathBuf::from( + std::env::args_os() + .nth(1) + .ok_or("expected a new SQLite file path")?, + ); + if !path.is_absolute() { + return Err("expected an absolute SQLite file path".into()); + } + OpenOptions::new() + .write(true) + .create_new(true) + .open(&path)?; + let url = format!( + "sqlite://{}", + path.to_str().ok_or("SQLite path must be UTF-8")? + ); + let db = Database::connect(ConnectOptions::new(url)).await?; + jupiter_migrate::apply_migrations(&db, false).await?; + db.close().await?; + println!("{}", path.display()); + Ok(()) +} diff --git a/jupiter-migrate/src/migration/m20260906_120000_snapshot_source_identity.rs b/jupiter-migrate/src/migration/m20260906_120000_snapshot_source_identity.rs new file mode 100644 index 000000000..cdf2d6d34 --- /dev/null +++ b/jupiter-migrate/src/migration/m20260906_120000_snapshot_source_identity.rs @@ -0,0 +1,132 @@ +//! Additive source identity/provenance storage. This does not enable namespace +//! publication or backfill historical scope claims automatically. + +use sea_orm_migration::{prelude::*, schema::*}; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .create_table( + Table::create() + .table(SnapshotInstance::Table) + .if_not_exists() + .col(string(SnapshotInstance::Singleton).primary_key()) + .col(string(SnapshotInstance::InstanceId).unique_key()) + .col(timestamp(SnapshotInstance::CreatedAt).default(Expr::current_timestamp())) + .to_owned(), + ) + .await?; + manager + .create_table( + Table::create() + .table(SnapshotSource::Table) + .if_not_exists() + .col(string(SnapshotSource::SourceId).primary_key()) + .col(string(SnapshotSource::InstanceId)) + .col(string(SnapshotSource::Kind)) + // Native source uses repo_id=0. Imported repo IDs are positive and + // never reused; this is not a cascading FK to the mutable registry. + .col(big_integer(SnapshotSource::RepoId)) + .col(timestamp(SnapshotSource::CreatedAt).default(Expr::current_timestamp())) + .index( + Index::create() + .name("uq_snapshot_source_backend") + .unique() + .col(SnapshotSource::InstanceId) + .col(SnapshotSource::Kind) + .col(SnapshotSource::RepoId), + ) + .foreign_key( + ForeignKey::create() + .name("fk_snapshot_source_instance") + .from(SnapshotSource::Table, SnapshotSource::InstanceId) + .to(SnapshotInstance::Table, SnapshotInstance::InstanceId) + .on_delete(ForeignKeyAction::Restrict), + ) + .to_owned(), + ) + .await?; + manager + .create_table( + Table::create() + .table(SourceCommitScope::Table) + .if_not_exists() + .col(string(SourceCommitScope::SourceId)) + // Index the digest, not up to 4096 UTF-8 path bytes: long path keys + // can exceed PostgreSQL's btree index tuple size limit. + .col(string(SourceCommitScope::ScopeKey)) + .col(text(SourceCommitScope::ScopePath)) + .col(string(SourceCommitScope::ObjectFormat)) + .col(string(SourceCommitScope::CommitOid)) + .col(string(SourceCommitScope::RootTreeOid)) + .col(string(SourceCommitScope::ProofKind)) + .col(string_null(SourceCommitScope::ProofOid)) + .col(timestamp(SourceCommitScope::CreatedAt).default(Expr::current_timestamp())) + .primary_key( + Index::create() + .col(SourceCommitScope::SourceId) + .col(SourceCommitScope::ScopeKey) + .col(SourceCommitScope::ObjectFormat) + .col(SourceCommitScope::CommitOid), + ) + .foreign_key( + ForeignKey::create() + .name("fk_scope_snapshot_source") + .from(SourceCommitScope::Table, SourceCommitScope::SourceId) + .to(SnapshotSource::Table, SnapshotSource::SourceId) + .on_delete(ForeignKeyAction::Restrict), + ) + .to_owned(), + ) + .await?; + Ok(()) + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + // Destructive schema rollback is only for an explicitly requested + // migration rollback. Application rollback must retain issued snapshots. + manager + .drop_table(Table::drop().table(SourceCommitScope::Table).to_owned()) + .await?; + manager + .drop_table(Table::drop().table(SnapshotSource::Table).to_owned()) + .await?; + manager + .drop_table(Table::drop().table(SnapshotInstance::Table).to_owned()) + .await + } +} + +#[derive(DeriveIden)] +enum SnapshotInstance { + Table, + Singleton, + InstanceId, + CreatedAt, +} +#[derive(DeriveIden)] +enum SnapshotSource { + Table, + SourceId, + InstanceId, + Kind, + RepoId, + CreatedAt, +} +#[derive(DeriveIden)] +enum SourceCommitScope { + Table, + SourceId, + ScopeKey, + ScopePath, + ObjectFormat, + CommitOid, + RootTreeOid, + ProofKind, + ProofOid, + CreatedAt, +} diff --git a/jupiter-migrate/src/migration/m20260906_140000_namespace_nodes.rs b/jupiter-migrate/src/migration/m20260906_140000_namespace_nodes.rs new file mode 100644 index 000000000..df52815c9 --- /dev/null +++ b/jupiter-migrate/src/migration/m20260906_140000_namespace_nodes.rs @@ -0,0 +1,45 @@ +//! Insert-only, content-addressed namespace index/value storage. No publication +//! head is created or enabled by this additive schema migration. + +use sea_orm_migration::{prelude::*, schema::*}; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .create_table( + Table::create() + .table(NamespaceNode::Table) + .if_not_exists() + .col(string(NamespaceNode::Digest).primary_key()) + .col(big_integer(NamespaceNode::SchemaVersion)) + .col(var_binary(NamespaceNode::CanonicalBytes, 16384)) + .col( + timestamp_with_time_zone(NamespaceNode::CreatedAt) + .default(Expr::current_timestamp()), + ) + .check(Expr::cust("length(canonical_bytes) <= 16384")) + .to_owned(), + ) + .await + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + // Explicit DDL rollback only. Application rollback must retain history. + manager + .drop_table(Table::drop().table(NamespaceNode::Table).to_owned()) + .await + } +} + +#[derive(DeriveIden)] +enum NamespaceNode { + Table, + Digest, + SchemaVersion, + CanonicalBytes, + CreatedAt, +} diff --git a/jupiter-migrate/src/migration/m20260906_145000_snapshot_utc_timestamps.rs b/jupiter-migrate/src/migration/m20260906_145000_snapshot_utc_timestamps.rs new file mode 100644 index 000000000..3dd34bbbf --- /dev/null +++ b/jupiter-migrate/src/migration/m20260906_145000_snapshot_utc_timestamps.rs @@ -0,0 +1,44 @@ +//! Align PostgreSQL columns with CLI-generated DateTimeUtc source entities. +//! Keep this as a forward migration: an earlier draft source schema may already +//! have been applied. Existing source code wrote UTC values into these columns. + +use sea_orm::DbBackend; +use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + if manager.get_database_backend() != DbBackend::Postgres { + return Ok(()); + } + for table in [ + "snapshot_instance", + "snapshot_source", + "source_commit_scope", + ] { + manager.get_connection().execute_unprepared(&format!( + "ALTER TABLE {table} ALTER COLUMN created_at TYPE TIMESTAMP WITH TIME ZONE USING created_at AT TIME ZONE 'UTC'" + )).await?; + } + Ok(()) + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + if manager.get_database_backend() != DbBackend::Postgres { + return Ok(()); + } + for table in [ + "snapshot_instance", + "snapshot_source", + "source_commit_scope", + ] { + manager.get_connection().execute_unprepared(&format!( + "ALTER TABLE {table} ALTER COLUMN created_at TYPE TIMESTAMP WITHOUT TIME ZONE USING created_at AT TIME ZONE 'UTC'" + )).await?; + } + Ok(()) + } +} diff --git a/jupiter-migrate/src/migration/m20260906_160000_namespace_publication.rs b/jupiter-migrate/src/migration/m20260906_160000_namespace_publication.rs new file mode 100644 index 000000000..d6fa99de0 --- /dev/null +++ b/jupiter-migrate/src/migration/m20260906_160000_namespace_publication.rs @@ -0,0 +1,133 @@ +//! Publication metadata only. No initial head, feature flag or historical +//! catalog is synthesized. All fields use portable SQLite/PostgreSQL types. + +use sea_orm_migration::{prelude::*, schema::*}; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .create_table( + Table::create() + .table(Meta::NamespaceView) + .if_not_exists() + .col(string(Meta::ViewId).primary_key()) + .col(string(Meta::InstanceId)) + .col(var_binary(Meta::CanonicalBytes, 16384)) + .col(timestamp_with_time_zone(Meta::CreatedAt)) + .check(Expr::cust("length(canonical_bytes) <= 16384")) + .to_owned(), + ) + .await?; + manager + .create_table( + Table::create() + .table(Meta::NamespaceHead) + .if_not_exists() + .col(string(Meta::InstanceId).primary_key()) + .col(big_integer(Meta::PublicationSeq)) + .col(string(Meta::ViewId)) + .col(big_integer(Meta::WriterEpoch)) + .check(Expr::cust("publication_seq > 0 AND writer_epoch > 0")) + .to_owned(), + ) + .await?; + manager.create_table(Table::create() + .table(Meta::NamespacePublication).if_not_exists() + .col(string(Meta::InstanceId)) + .col(big_integer(Meta::PublicationSeq)) + .col(string(Meta::ViewId)) + .col(ColumnDef::new(Meta::ParentSeq).big_integer().null()) + .col(ColumnDef::new(Meta::ParentViewId).string().null()) + .col(big_integer(Meta::WriterEpoch)) + .col(string(Meta::ActorDomain)) + .col(string(Meta::OperationId)) + .col(string(Meta::Reason)) + .col(timestamp_with_time_zone(Meta::CreatedAt)) + .primary_key(Index::create().col(Meta::InstanceId).col(Meta::PublicationSeq)) + .check(Expr::cust("publication_seq > 0 AND writer_epoch > 0")) + .check(Expr::cust("(parent_seq IS NULL AND parent_view_id IS NULL) OR (parent_seq IS NOT NULL AND parent_seq > 0 AND parent_view_id IS NOT NULL)")) + .to_owned()).await?; + manager.create_table(Table::create() + .table(Meta::SnapshotOperation).if_not_exists() + .col(string(Meta::ActorDomain)) + .col(string(Meta::OperationId)) + .col(string(Meta::InstanceId)) + .col(string(Meta::RequestDigest)) + .col(ColumnDef::new(Meta::PublicationSeq).big_integer().null()) + .col(ColumnDef::new(Meta::ViewId).string().null()) + .col(ColumnDef::new(Meta::Outcome).string().null()) + .col(timestamp_with_time_zone(Meta::CreatedAt)) + .primary_key(Index::create().col(Meta::ActorDomain).col(Meta::OperationId)) + .check(Expr::cust("(publication_seq IS NULL AND view_id IS NULL AND outcome IS NULL) OR (publication_seq IS NOT NULL AND publication_seq > 0 AND view_id IS NOT NULL AND outcome IS NOT NULL AND outcome IN ('published', 'no_op'))")) + .to_owned()).await?; + manager + .create_table( + Table::create() + .table(Meta::NamespaceOutbox) + .if_not_exists() + .col(string(Meta::EventId).primary_key()) + .col(string(Meta::InstanceId)) + .col(big_integer(Meta::PublicationSeq)) + .col(string(Meta::ViewId)) + .col(boolean(Meta::Delivered).default(false)) + .col(timestamp_with_time_zone(Meta::CreatedAt)) + .check(Expr::cust("publication_seq > 0")) + .to_owned(), + ) + .await?; + manager + .create_index( + Index::create() + .name("namespace_outbox_pending") + .table(Meta::NamespaceOutbox) + .col(Meta::Delivered) + .col(Meta::CreatedAt) + .to_owned(), + ) + .await + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + // Disposable-test DDL rollback only; application rollback retains history. + for table in [ + Meta::NamespaceOutbox, + Meta::SnapshotOperation, + Meta::NamespacePublication, + Meta::NamespaceHead, + Meta::NamespaceView, + ] { + manager + .drop_table(Table::drop().table(table).to_owned()) + .await?; + } + Ok(()) + } +} + +#[derive(DeriveIden)] +enum Meta { + NamespaceView, + NamespaceHead, + NamespacePublication, + SnapshotOperation, + NamespaceOutbox, + ViewId, + InstanceId, + CanonicalBytes, + CreatedAt, + PublicationSeq, + WriterEpoch, + ParentSeq, + ParentViewId, + ActorDomain, + OperationId, + Reason, + RequestDigest, + Outcome, + EventId, + Delivered, +} diff --git a/jupiter-migrate/src/migration/mod.rs b/jupiter-migrate/src/migration/mod.rs index b6f052c50..9f2c9449e 100644 --- a/jupiter-migrate/src/migration/mod.rs +++ b/jupiter-migrate/src/migration/mod.rs @@ -104,7 +104,13 @@ mod m20260723_080000_cla_sign_check_not_required; mod m20260804_120000_actor_to_campsite_user_id; mod m20260804_130000_data_backfill_ledger; mod m20260811_100000_create_campsite_member_identity; +mod m20260906_120000_snapshot_source_identity; +mod m20260906_140000_namespace_nodes; +mod m20260906_145000_snapshot_utc_timestamps; +mod m20260906_160000_namespace_publication; mod runner; +#[cfg(test)] +mod snapshot_tests; pub use runner::apply_migrations; /// Primary key `BIGINT` (not DB auto-increment); the application assigns `id` (e.g. `idgenerator::IdInstance::next_id`). @@ -195,6 +201,10 @@ impl MigratorTrait for Migrator { Box::new(m20260804_120000_actor_to_campsite_user_id::Migration), Box::new(m20260804_130000_data_backfill_ledger::Migration), Box::new(m20260811_100000_create_campsite_member_identity::Migration), + Box::new(m20260906_120000_snapshot_source_identity::Migration), + Box::new(m20260906_140000_namespace_nodes::Migration), + Box::new(m20260906_145000_snapshot_utc_timestamps::Migration), + Box::new(m20260906_160000_namespace_publication::Migration), ] } } diff --git a/jupiter-migrate/src/migration/snapshot_tests.rs b/jupiter-migrate/src/migration/snapshot_tests.rs new file mode 100644 index 000000000..e3afd543f --- /dev/null +++ b/jupiter-migrate/src/migration/snapshot_tests.rs @@ -0,0 +1,156 @@ +use callisto::git_repo; +use sea_orm::{ + ConnectOptions, ConnectionTrait, Database, DatabaseConnection, DbBackend, EntityTrait, Set, + Statement, +}; + +use super::*; + +const SNAPSHOT_TABLES: &[&str] = &[ + "snapshot_instance", + "snapshot_source", + "source_commit_scope", + "namespace_node", + "namespace_view", + "namespace_head", + "namespace_publication", + "snapshot_operation", + "namespace_outbox", +]; + +#[tokio::test] +async fn snapshot_migration_roundtrip_preserves_legacy_tables() { + let db = Database::connect( + ConnectOptions::new("sqlite::memory:") + .max_connections(1) + .to_owned(), + ) + .await + .unwrap(); + Migrator::up(&db, None).await.unwrap(); + let now = chrono::Utc::now().naive_utc(); + git_repo::Entity::insert(git_repo::ActiveModel { + id: Set(99), + repo_path: Set("/third-party/schema-fixture".into()), + repo_name: Set("schema-fixture".into()), + created_at: Set(now), + updated_at: Set(now), + }) + .exec(&db) + .await + .unwrap(); + for &name in SNAPSHOT_TABLES { + assert!(has_table(&db, name).await); + } + // Explicit destructive DDL rollback is exercised only on this private test + // database. Production application rollback must retain issued identities. + Migrator::down(&db, Some(4)).await.unwrap(); + assert!( + git_repo::Entity::find_by_id(99) + .one(&db) + .await + .unwrap() + .is_some() + ); + for &name in SNAPSHOT_TABLES { + assert!(!has_table(&db, name).await); + } + Migrator::up(&db, None).await.unwrap(); + Migrator::up(&db, None).await.unwrap(); + assert!( + git_repo::Entity::find_by_id(99) + .one(&db) + .await + .unwrap() + .is_some() + ); + for &name in SNAPSHOT_TABLES { + assert!(has_table(&db, name).await); + } +} + +async fn has_table(db: &DatabaseConnection, name: &str) -> bool { + db.query_one_raw(Statement::from_sql_and_values( + DbBackend::Sqlite, + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?", + [name.into()], + )) + .await + .unwrap() + .is_some() +} + +#[tokio::test] +#[ignore = "requires MEGA_SNAPSHOT_TEST_DATABASE_URL for a disposable loopback PostgreSQL test database"] +async fn snapshot_postgres_timestamp_forward_migration_preserves_existing_utc_values() { + let url = std::env::var("MEGA_SNAPSHOT_TEST_DATABASE_URL") + .expect("set explicit disposable PostgreSQL test URL"); + // This opt-in test only creates a fresh schema. It never drops/refreshes a + // supplied database or rewrites existing application schemas. + let parsed = url::Url::parse(&url).unwrap(); + assert!(matches!(parsed.scheme(), "postgres" | "postgresql")); + assert!(matches!( + parsed.host_str(), + Some("localhost" | "127.0.0.1" | "[::1]") + )); + assert_eq!( + parsed.path(), + "/snapshot_test", + "use the disposable snapshot_test database" + ); + let schema = format!("snapshot_time_test_{}", common::utils::generate_id()); + let control = Database::connect( + ConnectOptions::new(url.clone()) + .max_connections(1) + .sqlx_logging(false) + .to_owned(), + ) + .await + .unwrap(); + control + .execute_unprepared(&format!("CREATE SCHEMA {schema}")) + .await + .unwrap(); + let db = Database::connect( + ConnectOptions::new(url) + .max_connections(1) + .sqlx_logging(false) + .set_schema_search_path(schema.clone()) + .to_owned(), + ) + .await + .unwrap(); + let utc_index = Migrator::migrations() + .iter() + .position(|m| m.name() == "m20260906_145000_snapshot_utc_timestamps") + .unwrap(); + Migrator::up(&db, Some(utc_index as u32)).await.unwrap(); + db.execute_unprepared("INSERT INTO snapshot_instance (singleton, instance_id, created_at) VALUES ('utc-fixture', '11111111-1111-4111-8111-111111111111', TIMESTAMP '2024-01-02 03:04:05')").await.unwrap(); + db.execute_unprepared("SET TIME ZONE 'America/Los_Angeles'") + .await + .unwrap(); + Migrator::up(&db, Some(1)).await.unwrap(); + let model = callisto::snapshot_instance::Entity::find_by_id("utc-fixture".to_owned()) + .one(&db) + .await + .unwrap() + .unwrap(); + let expected = chrono::DateTime::parse_from_rfc3339("2024-01-02T03:04:05Z").unwrap(); + assert_eq!(model.created_at.timestamp(), expected.timestamp()); + Migrator::down(&db, Some(1)).await.unwrap(); + let row = db.query_one_raw(Statement::from_string(DbBackend::Postgres, "SELECT created_at::text AS stamp FROM snapshot_instance WHERE singleton = 'utc-fixture'")).await.unwrap().unwrap(); + assert_eq!( + row.try_get::("", "stamp").unwrap(), + "2024-01-02 03:04:05" + ); + Migrator::up(&db, None).await.unwrap(); + let model = callisto::snapshot_instance::Entity::find_by_id("utc-fixture".to_owned()) + .one(&db) + .await + .unwrap() + .unwrap(); + assert_eq!(model.created_at.timestamp(), expected.timestamp()); + println!( + "UTC timestamp forward/down/up verified under non-UTC session; retained test schema: {schema}" + ); +} diff --git a/jupiter/callisto/src/mod.rs b/jupiter/callisto/src/mod.rs index 17aa1536d..11deadafa 100644 --- a/jupiter/callisto/src/mod.rs +++ b/jupiter/callisto/src/mod.rs @@ -1,6 +1,15 @@ //! `SeaORM` Entity, @generated by sea-orm-codegen 2.0 +pub mod namespace_head; +pub mod namespace_node; +pub mod namespace_outbox; +pub mod namespace_publication; +pub mod namespace_view; pub mod prelude; +pub mod snapshot_instance; +pub mod snapshot_operation; +pub mod snapshot_source; +pub mod source_commit_scope; pub mod access_token; pub mod artifact_objects; diff --git a/jupiter/callisto/src/namespace_head.rs b/jupiter/callisto/src/namespace_head.rs new file mode 100644 index 000000000..ec2d9e486 --- /dev/null +++ b/jupiter/callisto/src/namespace_head.rs @@ -0,0 +1,17 @@ +//! `SeaORM` Entity, @generated by sea-orm-codegen 2.0 + +use sea_orm::entity::prelude::*; +use serde::{Deserialize, Serialize}; + +#[sea_orm::model] +#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel, Serialize, Deserialize)] +#[sea_orm(table_name = "namespace_head")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub instance_id: String, + pub publication_seq: i64, + pub view_id: String, + pub writer_epoch: i64, +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/jupiter/callisto/src/namespace_node.rs b/jupiter/callisto/src/namespace_node.rs new file mode 100644 index 000000000..f3c9b2024 --- /dev/null +++ b/jupiter/callisto/src/namespace_node.rs @@ -0,0 +1,18 @@ +//! `SeaORM` Entity, @generated by sea-orm-codegen 2.0 + +use sea_orm::entity::prelude::*; +use serde::{Deserialize, Serialize}; + +#[sea_orm::model] +#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel, Serialize, Deserialize)] +#[sea_orm(table_name = "namespace_node")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub digest: String, + pub schema_version: i64, + #[sea_orm(column_type = "VarBinary(StringLen::N(16384))")] + pub canonical_bytes: Vec, + pub created_at: DateTimeWithTimeZone, +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/jupiter/callisto/src/namespace_outbox.rs b/jupiter/callisto/src/namespace_outbox.rs new file mode 100644 index 000000000..cc0801847 --- /dev/null +++ b/jupiter/callisto/src/namespace_outbox.rs @@ -0,0 +1,19 @@ +//! `SeaORM` Entity, @generated by sea-orm-codegen 2.0 + +use sea_orm::entity::prelude::*; +use serde::{Deserialize, Serialize}; + +#[sea_orm::model] +#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel, Serialize, Deserialize)] +#[sea_orm(table_name = "namespace_outbox")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub event_id: String, + pub instance_id: String, + pub publication_seq: i64, + pub view_id: String, + pub delivered: bool, + pub created_at: DateTimeWithTimeZone, +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/jupiter/callisto/src/namespace_publication.rs b/jupiter/callisto/src/namespace_publication.rs new file mode 100644 index 000000000..95a5d5f86 --- /dev/null +++ b/jupiter/callisto/src/namespace_publication.rs @@ -0,0 +1,24 @@ +//! `SeaORM` Entity, @generated by sea-orm-codegen 2.0 + +use sea_orm::entity::prelude::*; +use serde::{Deserialize, Serialize}; + +#[sea_orm::model] +#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel, Serialize, Deserialize)] +#[sea_orm(table_name = "namespace_publication")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub instance_id: String, + #[sea_orm(primary_key, auto_increment = false)] + pub publication_seq: i64, + pub view_id: String, + pub parent_seq: Option, + pub parent_view_id: Option, + pub writer_epoch: i64, + pub actor_domain: String, + pub operation_id: String, + pub reason: String, + pub created_at: DateTimeWithTimeZone, +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/jupiter/callisto/src/namespace_view.rs b/jupiter/callisto/src/namespace_view.rs new file mode 100644 index 000000000..850c5e933 --- /dev/null +++ b/jupiter/callisto/src/namespace_view.rs @@ -0,0 +1,18 @@ +//! `SeaORM` Entity, @generated by sea-orm-codegen 2.0 + +use sea_orm::entity::prelude::*; +use serde::{Deserialize, Serialize}; + +#[sea_orm::model] +#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel, Serialize, Deserialize)] +#[sea_orm(table_name = "namespace_view")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub view_id: String, + pub instance_id: String, + #[sea_orm(column_type = "VarBinary(StringLen::N(16384))")] + pub canonical_bytes: Vec, + pub created_at: DateTimeWithTimeZone, +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/jupiter/callisto/src/prelude.rs b/jupiter/callisto/src/prelude.rs index 6c6317aef..55a6ea94f 100644 --- a/jupiter/callisto/src/prelude.rs +++ b/jupiter/callisto/src/prelude.rs @@ -29,10 +29,15 @@ pub use super::{ mega_tag::Entity as MegaTag, mega_tree::Entity as MegaTree, mega_webhook::Entity as MegaWebhook, mega_webhook_delivery::Entity as MegaWebhookDelivery, mega_webhook_event_type::Entity as MegaWebhookEventType, merge_queue::Entity as MergeQueue, + namespace_head::Entity as NamespaceHead, namespace_node::Entity as NamespaceNode, + namespace_outbox::Entity as NamespaceOutbox, + namespace_publication::Entity as NamespacePublication, namespace_view::Entity as NamespaceView, non_member_note_views::Entity as NonMemberNoteViews, note_views::Entity as NoteViews, notes::Entity as Notes, notification_event_types::Entity as NotificationEventTypes, orion_tasks::Entity as OrionTasks, path_check_configs::Entity as PathCheckConfigs, - reactions::Entity as Reactions, ssh_keys::Entity as SshKeys, + reactions::Entity as Reactions, snapshot_instance::Entity as SnapshotInstance, + snapshot_operation::Entity as SnapshotOperation, snapshot_source::Entity as SnapshotSource, + source_commit_scope::Entity as SourceCommitScope, ssh_keys::Entity as SshKeys, target_build_status::Entity as TargetBuildStatus, target_state_histories::Entity as TargetStateHistories, user_approval_status::Entity as UserApprovalStatus, diff --git a/jupiter/callisto/src/snapshot_instance.rs b/jupiter/callisto/src/snapshot_instance.rs new file mode 100644 index 000000000..f417732e6 --- /dev/null +++ b/jupiter/callisto/src/snapshot_instance.rs @@ -0,0 +1,19 @@ +//! `SeaORM` Entity, @generated by sea-orm-codegen 2.0 + +use sea_orm::entity::prelude::*; +use serde::{Deserialize, Serialize}; + +#[sea_orm::model] +#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel, Serialize, Deserialize)] +#[sea_orm(table_name = "snapshot_instance")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub singleton: String, + #[sea_orm(unique)] + pub instance_id: String, + pub created_at: DateTimeUtc, + #[sea_orm(has_many)] + pub snapshot_sources: HasMany, +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/jupiter/callisto/src/snapshot_operation.rs b/jupiter/callisto/src/snapshot_operation.rs new file mode 100644 index 000000000..83b810099 --- /dev/null +++ b/jupiter/callisto/src/snapshot_operation.rs @@ -0,0 +1,22 @@ +//! `SeaORM` Entity, @generated by sea-orm-codegen 2.0 + +use sea_orm::entity::prelude::*; +use serde::{Deserialize, Serialize}; + +#[sea_orm::model] +#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel, Serialize, Deserialize)] +#[sea_orm(table_name = "snapshot_operation")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub actor_domain: String, + #[sea_orm(primary_key, auto_increment = false)] + pub operation_id: String, + pub instance_id: String, + pub request_digest: String, + pub publication_seq: Option, + pub view_id: Option, + pub outcome: Option, + pub created_at: DateTimeWithTimeZone, +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/jupiter/callisto/src/snapshot_source.rs b/jupiter/callisto/src/snapshot_source.rs new file mode 100644 index 000000000..9295d8880 --- /dev/null +++ b/jupiter/callisto/src/snapshot_source.rs @@ -0,0 +1,28 @@ +//! `SeaORM` Entity, @generated by sea-orm-codegen 2.0 + +use sea_orm::entity::prelude::*; +use serde::{Deserialize, Serialize}; + +#[sea_orm::model] +#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel, Serialize, Deserialize)] +#[sea_orm(table_name = "snapshot_source")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub source_id: String, + pub instance_id: String, + pub kind: String, + pub repo_id: i64, + pub created_at: DateTimeUtc, + #[sea_orm( + belongs_to, + from = "instance_id", + to = "instance_id", + on_update = "NoAction", + on_delete = "Restrict" + )] + pub snapshot_instance: BelongsTo, + #[sea_orm(has_many)] + pub source_commit_scopes: HasMany, +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/jupiter/callisto/src/source_commit_scope.rs b/jupiter/callisto/src/source_commit_scope.rs new file mode 100644 index 000000000..37d8c9d6a --- /dev/null +++ b/jupiter/callisto/src/source_commit_scope.rs @@ -0,0 +1,34 @@ +//! `SeaORM` Entity, @generated by sea-orm-codegen 2.0 + +use sea_orm::entity::prelude::*; +use serde::{Deserialize, Serialize}; + +#[sea_orm::model] +#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel, Serialize, Deserialize)] +#[sea_orm(table_name = "source_commit_scope")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub source_id: String, + #[sea_orm(primary_key, auto_increment = false)] + pub scope_key: String, + #[sea_orm(column_type = "Text")] + pub scope_path: String, + #[sea_orm(primary_key, auto_increment = false)] + pub object_format: String, + #[sea_orm(primary_key, auto_increment = false)] + pub commit_oid: String, + pub root_tree_oid: String, + pub proof_kind: String, + pub proof_oid: Option, + pub created_at: DateTimeUtc, + #[sea_orm( + belongs_to, + from = "source_id", + to = "source_id", + on_update = "NoAction", + on_delete = "Restrict" + )] + pub snapshot_source: BelongsTo, +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/jupiter/src/storage/git_db_storage.rs b/jupiter/src/storage/git_db_storage.rs index fcf963b6e..3eaab0f8d 100644 --- a/jupiter/src/storage/git_db_storage.rs +++ b/jupiter/src/storage/git_db_storage.rs @@ -12,7 +12,7 @@ use common::{ use futures::Stream; use sea_orm::{ ActiveModelTrait, ColumnTrait, ConnectionTrait, DatabaseTransaction, DbErr, EntityTrait, - IntoActiveModel, PaginatorTrait, QueryFilter, QueryOrder, Set, TransactionTrait, + IntoActiveModel, PaginatorTrait, QueryFilter, QueryOrder, QuerySelect, Set, TransactionTrait, sea_query::{CaseStatement, Expr, ExprTrait, OnConflict}, }; @@ -31,6 +31,16 @@ impl Deref for GitDbStorage { } impl GitDbStorage { + /// Stable backend lookup, independent of the current namespace path. + pub async fn find_git_repo_by_id( + &self, + repo_id: i64, + ) -> Result, MegaError> { + Ok(git_repo::Entity::find_by_id(repo_id) + .one(self.get_connection()) + .await?) + } + pub async fn create_repo_and_save_ref( &self, repo_path: &str, @@ -116,6 +126,39 @@ impl GitDbStorage { Ok(result) } + /// Resolve one fully qualified ref without listing every ref in the repository. + pub async fn get_ref_by_name( + &self, + repo_id: i64, + ref_name: &str, + ) -> Result, MegaError> { + Ok(import_refs::Entity::find() + .filter(import_refs::Column::RepoId.eq(repo_id)) + .filter(import_refs::Column::RefName.eq(ref_name)) + .one(self.get_connection()) + .await?) + } + + /// Snapshot resolution must not arbitrarily select from ambiguous default refs. + /// Bound the query even when legacy metadata contains many default flags. + pub async fn get_unique_default_ref( + &self, + repo_id: i64, + ) -> Result, MegaError> { + let mut refs = import_refs::Entity::find() + .filter(import_refs::Column::RepoId.eq(repo_id)) + .filter(import_refs::Column::DefaultBranch.eq(true)) + .limit(2) + .all(self.get_connection()) + .await?; + if refs.len() > 1 { + return Err(MegaError::Conflict( + "multiple default refs in import repository".into(), + )); + } + Ok(refs.pop()) + } + pub async fn update_ref( &self, repo_id: i64, @@ -203,6 +246,30 @@ impl GitDbStorage { Ok(()) } + /// Conditional ref mutation for a caller-owned publication transaction. + /// Does not retry/rebase or silently replace a newer advertised-old value. + pub async fn update_ref_if_unchanged( + &self, + repo_id: i64, + ref_name: &str, + expected_git_id: &str, + new_git_id: &str, + conn: &C, + ) -> Result { + let result = import_refs::Entity::update_many() + .col_expr(import_refs::Column::RefGitId, Expr::value(new_git_id)) + .col_expr( + import_refs::Column::UpdatedAt, + Expr::value(chrono::Utc::now().naive_utc()), + ) + .filter(import_refs::Column::RepoId.eq(repo_id)) + .filter(import_refs::Column::RefName.eq(ref_name)) + .filter(import_refs::Column::RefGitId.eq(expected_git_id)) + .exec(conn) + .await?; + Ok(result.rows_affected == 1) + } + pub async fn get_default_ref( &self, repo_id: i64, diff --git a/jupiter/src/storage/mod.rs b/jupiter/src/storage/mod.rs index 03d583d6d..4f1eb7d8b 100644 --- a/jupiter/src/storage/mod.rs +++ b/jupiter/src/storage/mod.rs @@ -25,7 +25,10 @@ pub mod mono_storage; pub mod notification_storage; pub mod user_approval_storage; pub use notification_storage::NotificationStorage; +pub mod namespace_storage; pub mod note_storage; +pub mod publication_storage; +pub mod snapshot_storage; pub mod stg_common; pub mod user_storage; pub mod vault_storage; @@ -323,6 +326,24 @@ impl Storage { self.app_service.mono_storage.clone() } + pub fn snapshot_storage(&self) -> snapshot_storage::SnapshotStorage { + snapshot_storage::SnapshotStorage { + base: self.app_service.mono_storage.base.clone(), + } + } + + pub fn namespace_storage(&self) -> namespace_storage::NamespaceStorage { + namespace_storage::NamespaceStorage { + base: self.app_service.mono_storage.base.clone(), + } + } + + pub fn publication_storage(&self) -> publication_storage::PublicationStorage { + publication_storage::PublicationStorage { + base: self.app_service.mono_storage.base.clone(), + } + } + /// Begin a database transaction on the shared app connection (monorepo + import metadata). pub async fn begin_db_transaction(&self) -> Result { use sea_orm::TransactionTrait; diff --git a/jupiter/src/storage/mono_storage.rs b/jupiter/src/storage/mono_storage.rs index 687849210..84da4bde3 100644 --- a/jupiter/src/storage/mono_storage.rs +++ b/jupiter/src/storage/mono_storage.rs @@ -810,8 +810,7 @@ impl MonoStorage { Ok(mega_commit::Entity::find() .filter(mega_commit::Column::CommitId.eq(hash)) .one(self.get_connection()) - .await - .unwrap()) + .await?) } pub async fn get_commits_by_hashes( @@ -832,8 +831,7 @@ impl MonoStorage { Ok(mega_tree::Entity::find() .filter(mega_tree::Column::TreeId.eq(hash)) .one(self.get_connection()) - .await - .unwrap()) + .await?) } pub async fn get_trees_by_hashes( diff --git a/jupiter/src/storage/namespace_storage.rs b/jupiter/src/storage/namespace_storage.rs new file mode 100644 index 000000000..1f7b25eef --- /dev/null +++ b/jupiter/src/storage/namespace_storage.rs @@ -0,0 +1,106 @@ +//! Immutable namespace content storage. Publication policy/encoding belongs to +//! Ceres; this boundary enforces size, schema, digest and insert-only durability. + +use callisto::namespace_node; +use common::errors::MegaError; +use sea_orm::{ConnectionTrait, DbErr, EntityTrait, Set, sea_query::OnConflict}; +use sha2::{Digest, Sha256}; + +use super::base_storage::{BaseStorage, StorageConnector}; + +pub const MAX_NAMESPACE_NODE_BYTES: usize = 16 * 1024; + +#[derive(Clone)] +pub struct NamespaceStorage { + pub base: BaseStorage, +} + +impl NamespaceStorage { + pub async fn node(&self, digest: &str) -> Result>, MegaError> { + self.node_in(self.base.get_connection(), digest).await + } + + pub async fn node_in( + &self, + conn: &C, + digest: &str, + ) -> Result>, MegaError> { + validate_digest(digest)?; + let Some(node) = namespace_node::Entity::find_by_id(digest.to_owned()) + .one(conn) + .await? + else { + return Ok(None); + }; + if node.schema_version != 1 + || node.canonical_bytes.len() > MAX_NAMESPACE_NODE_BYTES + || node_digest(&node.canonical_bytes) != digest + { + return Err(MegaError::Unavailable( + "corrupt or unsupported namespace node".into(), + )); + } + Ok(Some(node.canonical_bytes)) + } + + /// Accept a caller transaction so immutable nodes and publication metadata + /// can commit together. Failed prepares never mutate an old view's bytes. + pub async fn put_node_in( + &self, + conn: &C, + digest: &str, + bytes: &[u8], + ) -> Result<(), MegaError> { + validate_digest(digest)?; + if bytes.len() > MAX_NAMESPACE_NODE_BYTES || node_digest(bytes) != digest { + return Err(MegaError::bad_request( + "invalid namespace node size or digest", + )); + } + let result = namespace_node::Entity::insert(namespace_node::ActiveModel { + digest: Set(digest.into()), + schema_version: Set(1), + canonical_bytes: Set(bytes.to_vec()), + created_at: Set(chrono::Utc::now().fixed_offset()), + }) + .on_conflict( + OnConflict::column(namespace_node::Column::Digest) + .do_nothing() + .to_owned(), + ) + .exec(conn) + .await; + match result { + Ok(_) | Err(DbErr::RecordNotInserted) => {} + Err(error) => return Err(error.into()), + } + let stored = self + .node_in(conn, digest) + .await? + .ok_or_else(|| MegaError::Unavailable("namespace node disappeared".into()))?; + if stored != bytes { + return Err(MegaError::Conflict( + "immutable namespace node mismatch".into(), + )); + } + Ok(()) + } +} + +pub fn node_digest(bytes: &[u8]) -> String { + format!("sha256:{}", hex::encode(Sha256::digest(bytes))) +} + +pub(super) fn validate_digest(digest: &str) -> Result<(), MegaError> { + if !digest.strip_prefix("sha256:").is_some_and(|s| { + s.len() == 64 + && s.bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + }) { + return Err(MegaError::bad_request("invalid namespace digest")); + } + Ok(()) +} + +#[cfg(test)] +mod tests; diff --git a/jupiter/src/storage/namespace_storage/tests.rs b/jupiter/src/storage/namespace_storage/tests.rs new file mode 100644 index 000000000..da8c13f0f --- /dev/null +++ b/jupiter/src/storage/namespace_storage/tests.rs @@ -0,0 +1,109 @@ +use callisto::namespace_node; +use sea_orm::{ColumnTrait, QueryFilter, TransactionTrait, sea_query::Expr}; +use tempfile::TempDir; + +use super::*; + +#[tokio::test] +async fn namespace_nodes_are_idempotent_bounded_and_transactional() { + let dir = TempDir::new().unwrap(); + let storage = crate::tests::test_storage(dir.path()).await; + let nodes = storage.namespace_storage(); + let conn = nodes.base.get_connection(); + let bytes = b"canonical fixture"; + let digest = node_digest(bytes); + let txn = conn.begin().await.unwrap(); + nodes.put_node_in(&txn, &digest, bytes).await.unwrap(); + assert_eq!(nodes.node_in(&txn, &digest).await.unwrap().unwrap(), bytes); + txn.rollback().await.unwrap(); + assert!(nodes.node(&digest).await.unwrap().is_none()); + let txn = conn.begin().await.unwrap(); + nodes.put_node_in(&txn, &digest, bytes).await.unwrap(); + nodes.put_node_in(&txn, &digest, bytes).await.unwrap(); + txn.commit().await.unwrap(); + assert_eq!(nodes.node(&digest).await.unwrap().unwrap(), bytes); + assert!( + nodes + .put_node_in(conn, &digest, b"different") + .await + .is_err() + ); + let max = vec![42; MAX_NAMESPACE_NODE_BYTES]; + nodes + .put_node_in(conn, &node_digest(&max), &max) + .await + .unwrap(); + let too_large = vec![42; MAX_NAMESPACE_NODE_BYTES + 1]; + assert!( + nodes + .put_node_in(conn, &node_digest(&too_large), &too_large) + .await + .is_err() + ); + // The database constraint also protects callers that bypass the facade. + assert!( + namespace_node::Entity::insert(namespace_node::ActiveModel { + digest: Set(node_digest(&too_large)), + schema_version: Set(1), + canonical_bytes: Set(too_large), + created_at: Set(chrono::Utc::now().fixed_offset()), + }) + .exec(conn) + .await + .is_err() + ); + assert!(nodes.node("sha1:bad").await.is_err()); +} + +#[tokio::test] +async fn namespace_nodes_reject_corruption_and_unknown_schema() { + let dir = TempDir::new().unwrap(); + let storage = crate::tests::test_storage(dir.path()).await; + let nodes = storage.namespace_storage(); + let conn = nodes.base.get_connection(); + let bytes = b"canonical fixture"; + let digest = node_digest(bytes); + nodes.put_node_in(conn, &digest, bytes).await.unwrap(); + namespace_node::Entity::update_many() + .col_expr( + namespace_node::Column::CanonicalBytes, + Expr::value(b"corrupt".to_vec()), + ) + .filter(namespace_node::Column::Digest.eq(&digest)) + .exec(conn) + .await + .unwrap(); + assert!(matches!( + nodes.node(&digest).await, + Err(MegaError::Unavailable(_)) + )); + assert!(nodes.put_node_in(conn, &digest, bytes).await.is_err()); + namespace_node::Entity::update_many() + .col_expr( + namespace_node::Column::CanonicalBytes, + Expr::value(bytes.to_vec()), + ) + .col_expr(namespace_node::Column::SchemaVersion, Expr::value(2i64)) + .filter(namespace_node::Column::Digest.eq(&digest)) + .exec(conn) + .await + .unwrap(); + assert!(nodes.node(&digest).await.is_err()); +} + +#[tokio::test] +async fn concurrent_identical_namespace_inserts_return_the_same_bytes() { + let dir = TempDir::new().unwrap(); + let storage = crate::tests::test_storage(dir.path()).await; + let nodes = storage.namespace_storage(); + let bytes = b"shared"; + let digest = node_digest(bytes); + let results = futures::future::join_all( + (0..16).map(|_| nodes.put_node_in(nodes.base.get_connection(), &digest, bytes)), + ) + .await; + for result in results { + result.unwrap(); + } + assert_eq!(nodes.node(&digest).await.unwrap().unwrap(), bytes); +} diff --git a/jupiter/src/storage/publication_storage.rs b/jupiter/src/storage/publication_storage.rs new file mode 100644 index 000000000..b0d9c7eed --- /dev/null +++ b/jupiter/src/storage/publication_storage.rs @@ -0,0 +1,506 @@ +//! Transaction owner for namespace metadata. Reserve an operation BEFORE any +//! ref write; either return its committed receipt or lend the same transaction +//! to the writer. This core does not authorize writers, verify Git objects, +//! apply binding policy, or supply retention pins. Ceres must provide those +//! gates before a production capability can be enabled. + +use callisto::{ + namespace_head, namespace_outbox, namespace_publication, namespace_view, snapshot_instance, + snapshot_operation, +}; +use common::errors::MegaError; +use sea_orm::{ + ColumnTrait, ConnectionTrait, DatabaseTransaction, DbErr, EntityTrait, QueryFilter, Set, + TransactionTrait, + sea_query::{Expr, OnConflict}, +}; + +use super::{ + base_storage::{BaseStorage, StorageConnector}, + namespace_storage::{MAX_NAMESPACE_NODE_BYTES, node_digest, validate_digest}, +}; + +#[derive(Clone)] +pub struct PublicationStorage { + pub base: BaseStorage, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PublicationHead { + pub publication_seq: i64, + pub view_id: String, + pub writer_epoch: i64, +} + +/// The authenticated caller supplies actor_domain; it is not a client-selected +/// authorization grant. request_digest must cover the complete canonical plan, +/// including expected refs/head, binding policy and prepared content identities. +#[derive(Debug, Clone)] +pub struct PublicationRequest { + pub instance_id: String, + pub actor_domain: String, + pub operation_id: String, + pub request_digest: String, +} + +/// Ceres supplies verified namespace-manifest-v1 bytes. Jupiter checks the byte +/// bound, identity and immutability, not the higher-level descriptor semantics. +#[derive(Debug, Clone)] +pub struct PreparedNamespaceView { + pub instance_id: String, + pub view_id: String, + pub canonical_bytes: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PublicationOutcome { + Published, + NoOp, +} + +impl PublicationOutcome { + fn as_str(self) -> &'static str { + match self { + Self::Published => "published", + Self::NoOp => "no_op", + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PublicationReceipt { + pub publication_seq: i64, + pub view_id: String, + pub outcome: PublicationOutcome, +} + +pub enum BeginPublication { + Replay(PublicationReceipt), + Ready(Box), +} + +/// Never expose ownership of the underlying transaction: only finish can +/// commit the reserved operation, and abort/drop rolls back staged ref writes. +pub struct PublicationTransaction { + storage: PublicationStorage, + transaction: DatabaseTransaction, + request: PublicationRequest, + expected: Option, + writer_epoch: i64, +} + +impl PublicationStorage { + pub async fn head(&self, instance: &str) -> Result, MegaError> { + self.head_in(self.base.get_connection(), instance).await + } + + async fn head_in( + &self, + conn: &C, + instance: &str, + ) -> Result, MegaError> { + validate_uuid(instance)?; + let row = namespace_head::Entity::find_by_id(instance.to_owned()) + .one(conn) + .await?; + row.map(|row| { + let head = PublicationHead { + publication_seq: row.publication_seq, + view_id: row.view_id, + writer_epoch: row.writer_epoch, + }; + validate_head(&head)?; + Ok(head) + }) + .transpose() + } + + pub async fn view(&self, instance: &str, id: &str) -> Result>, MegaError> { + validate_uuid(instance)?; + validate_digest(id)?; + let row = namespace_view::Entity::find_by_id(id.to_owned()) + .filter(namespace_view::Column::InstanceId.eq(instance)) + .one(self.base.get_connection()) + .await?; + row.map(|row| { + validate_view_bytes(&row.view_id, &row.canonical_bytes)?; + Ok(row.canonical_bytes) + }) + .transpose() + } + + /// Receipt lookup after a lost response or uncertain COMMIT. The endpoint + /// must independently authenticate the same actor domain on every lookup. + pub async fn receipt( + &self, + request: &PublicationRequest, + ) -> Result, MegaError> { + validate_request(request)?; + let row = snapshot_operation::Entity::find_by_id(( + request.actor_domain.clone(), + request.operation_id.clone(), + )) + .one(self.base.get_connection()) + .await?; + row.map(|row| receipt_from(row, request)).transpose() + } + + pub async fn begin( + &self, + request: PublicationRequest, + expected: Option, + writer_epoch: i64, + ) -> Result { + validate_request(&request)?; + if writer_epoch <= 0 { + return Err(MegaError::bad_request("invalid writer epoch")); + } + if let Some(head) = &expected { + validate_head(head)?; + if head.writer_epoch != writer_epoch { + return Err(MegaError::Conflict("writer epoch mismatch".into())); + } + } + let transaction = self.base.get_connection().begin().await?; + match self.reserve(&transaction, &request, &expected).await { + Ok(Some(receipt)) => { + transaction.rollback().await?; + Ok(BeginPublication::Replay(receipt)) + } + Ok(None) => Ok(BeginPublication::Ready(Box::new(PublicationTransaction { + storage: self.clone(), + transaction, + request, + expected, + writer_epoch, + }))), + Err(error) => { + transaction.rollback().await?; + Err(error) + } + } + } + + async fn reserve( + &self, + txn: &DatabaseTransaction, + request: &PublicationRequest, + expected: &Option, + ) -> Result, MegaError> { + // A unique insert serializes duplicate operations even on PostgreSQL. + // SQLite obtains its write reservation before reading mutable head state. + let result = snapshot_operation::Entity::insert(snapshot_operation::ActiveModel { + actor_domain: Set(request.actor_domain.clone()), + operation_id: Set(request.operation_id.clone()), + instance_id: Set(request.instance_id.clone()), + request_digest: Set(request.request_digest.clone()), + publication_seq: Set(None), + view_id: Set(None), + outcome: Set(None), + created_at: Set(chrono::Utc::now().fixed_offset()), + }) + .on_conflict( + OnConflict::columns([ + snapshot_operation::Column::ActorDomain, + snapshot_operation::Column::OperationId, + ]) + .do_nothing() + .to_owned(), + ) + .exec(txn) + .await; + match result { + Ok(_) => {} + Err(DbErr::RecordNotInserted) => { + let row = snapshot_operation::Entity::find_by_id(( + request.actor_domain.clone(), + request.operation_id.clone(), + )) + .one(txn) + .await? + .ok_or_else(|| MegaError::Unavailable("operation disappeared".into()))?; + return receipt_from(row, request).map(Some); + } + Err(error) => return Err(error.into()), + } + let registered = snapshot_instance::Entity::find() + .filter(snapshot_instance::Column::InstanceId.eq(&request.instance_id)) + .one(txn) + .await?; + if registered.is_none() { + return Err(MegaError::NotFound( + "snapshot instance not registered".into(), + )); + } + if &self.head_in(txn, &request.instance_id).await? != expected { + return Err(MegaError::Conflict( + "expected namespace head mismatch".into(), + )); + } + Ok(None) + } + + async fn put_view( + &self, + txn: &DatabaseTransaction, + view: &PreparedNamespaceView, + ) -> Result<(), MegaError> { + validate_view_bytes(&view.view_id, &view.canonical_bytes)?; + let inserted = namespace_view::Entity::insert(namespace_view::ActiveModel { + view_id: Set(view.view_id.clone()), + instance_id: Set(view.instance_id.clone()), + canonical_bytes: Set(view.canonical_bytes.clone()), + created_at: Set(chrono::Utc::now().fixed_offset()), + }) + .on_conflict( + OnConflict::column(namespace_view::Column::ViewId) + .do_nothing() + .to_owned(), + ) + .exec(txn) + .await; + match inserted { + Ok(_) | Err(DbErr::RecordNotInserted) => {} + Err(e) => return Err(e.into()), + } + let stored = namespace_view::Entity::find_by_id(view.view_id.clone()) + .one(txn) + .await? + .ok_or_else(|| MegaError::Unavailable("namespace view disappeared".into()))?; + if stored.instance_id != view.instance_id || stored.canonical_bytes != view.canonical_bytes + { + return Err(MegaError::Conflict( + "immutable namespace view mismatch".into(), + )); + } + Ok(()) + } +} + +impl PublicationTransaction { + pub fn transaction(&self) -> &DatabaseTransaction { + &self.transaction + } + + pub async fn abort(self) -> Result<(), MegaError> { + self.transaction.rollback().await?; + Ok(()) + } + + pub async fn finish( + self, + view: &PreparedNamespaceView, + reason: &str, + ) -> Result { + let receipt = match self.stage_finish(view, reason).await { + Ok(receipt) => receipt, + Err(error) => { + self.transaction.rollback().await?; + return Err(error); + } + }; + self.transaction.commit().await.map_err(|_| { + MegaError::Unavailable( + "publication commit outcome uncertain; query operation receipt before retrying" + .into(), + ) + })?; + Ok(receipt) + } + + async fn stage_finish( + &self, + view: &PreparedNamespaceView, + reason: &str, + ) -> Result { + if view.instance_id != self.request.instance_id { + return Err(MegaError::bad_request("view instance mismatch")); + } + validate_label(reason)?; + let changed = self + .expected + .as_ref() + .is_none_or(|head| head.view_id != view.view_id); + let old_seq = self + .expected + .as_ref() + .map_or(0, |head| head.publication_seq); + let seq = if changed { + old_seq + .checked_add(1) + .ok_or_else(|| MegaError::Unavailable("publication sequence exhausted".into()))? + } else { + old_seq + }; + self.storage.put_view(&self.transaction, view).await?; + self.cas_head(seq, &view.view_id).await?; + let now = chrono::Utc::now().fixed_offset(); + if changed { + namespace_publication::Entity::insert(namespace_publication::ActiveModel { + instance_id: Set(self.request.instance_id.clone()), + publication_seq: Set(seq), + view_id: Set(view.view_id.clone()), + parent_seq: Set(self.expected.as_ref().map(|h| h.publication_seq)), + parent_view_id: Set(self.expected.as_ref().map(|h| h.view_id.clone())), + writer_epoch: Set(self.writer_epoch), + actor_domain: Set(self.request.actor_domain.clone()), + operation_id: Set(self.request.operation_id.clone()), + reason: Set(reason.into()), + created_at: Set(now), + }) + .exec(&self.transaction) + .await?; + namespace_outbox::Entity::insert(namespace_outbox::ActiveModel { + event_id: Set(uuid::Uuid::new_v4().to_string()), + instance_id: Set(self.request.instance_id.clone()), + publication_seq: Set(seq), + view_id: Set(view.view_id.clone()), + delivered: Set(false), + created_at: Set(now), + }) + .exec(&self.transaction) + .await?; + } + let outcome = if changed { + PublicationOutcome::Published + } else { + PublicationOutcome::NoOp + }; + let updated = snapshot_operation::Entity::update_many() + .col_expr(snapshot_operation::Column::PublicationSeq, Expr::value(seq)) + .col_expr( + snapshot_operation::Column::ViewId, + Expr::value(view.view_id.clone()), + ) + .col_expr( + snapshot_operation::Column::Outcome, + Expr::value(outcome.as_str()), + ) + .filter(snapshot_operation::Column::ActorDomain.eq(&self.request.actor_domain)) + .filter(snapshot_operation::Column::OperationId.eq(&self.request.operation_id)) + .filter(snapshot_operation::Column::RequestDigest.eq(&self.request.request_digest)) + .filter(snapshot_operation::Column::Outcome.is_null()) + .exec(&self.transaction) + .await?; + if updated.rows_affected != 1 { + return Err(MegaError::Conflict("operation reservation changed".into())); + } + Ok(PublicationReceipt { + publication_seq: seq, + view_id: view.view_id.clone(), + outcome, + }) + } + + async fn cas_head(&self, seq: i64, view_id: &str) -> Result<(), MegaError> { + if let Some(expected) = &self.expected { + // Even a no-op performs the fence; stale writers cannot commit ref + // mutations simply because their proposed view did not change. + let updated = namespace_head::Entity::update_many() + .col_expr(namespace_head::Column::PublicationSeq, Expr::value(seq)) + .col_expr(namespace_head::Column::ViewId, Expr::value(view_id)) + .filter(namespace_head::Column::InstanceId.eq(&self.request.instance_id)) + .filter(namespace_head::Column::PublicationSeq.eq(expected.publication_seq)) + .filter(namespace_head::Column::ViewId.eq(&expected.view_id)) + .filter(namespace_head::Column::WriterEpoch.eq(self.writer_epoch)) + .exec(&self.transaction) + .await?; + if updated.rows_affected != 1 { + return Err(MegaError::Conflict( + "expected namespace head mismatch".into(), + )); + } + } else { + let inserted = namespace_head::Entity::insert(namespace_head::ActiveModel { + instance_id: Set(self.request.instance_id.clone()), + publication_seq: Set(seq), + view_id: Set(view_id.into()), + writer_epoch: Set(self.writer_epoch), + }) + .on_conflict( + OnConflict::column(namespace_head::Column::InstanceId) + .do_nothing() + .to_owned(), + ) + .exec(&self.transaction) + .await; + match inserted { + Ok(_) => {} + Err(DbErr::RecordNotInserted) => { + return Err(MegaError::Conflict( + "namespace head already initialized".into(), + )); + } + Err(e) => return Err(e.into()), + } + } + Ok(()) + } +} + +fn validate_request(request: &PublicationRequest) -> Result<(), MegaError> { + validate_uuid(&request.instance_id)?; + validate_uuid(&request.operation_id)?; + validate_label(&request.actor_domain)?; + validate_digest(&request.request_digest) +} +fn validate_uuid(value: &str) -> Result<(), MegaError> { + if !uuid::Uuid::parse_str(value).is_ok_and(|id| !id.is_nil() && id.to_string() == value) { + return Err(MegaError::bad_request("invalid publication UUID")); + } + Ok(()) +} +fn validate_label(value: &str) -> Result<(), MegaError> { + if value.is_empty() || value.len() > 255 || value.chars().any(char::is_control) { + return Err(MegaError::bad_request("invalid publication label")); + } + Ok(()) +} +fn validate_head(head: &PublicationHead) -> Result<(), MegaError> { + if head.publication_seq <= 0 || head.writer_epoch <= 0 { + return Err(MegaError::bad_request("invalid publication head counters")); + } + validate_digest(&head.view_id) +} +fn validate_view_bytes(id: &str, bytes: &[u8]) -> Result<(), MegaError> { + validate_digest(id)?; + if bytes.len() > MAX_NAMESPACE_NODE_BYTES || node_digest(bytes) != id { + return Err(MegaError::Unavailable( + "invalid namespace view content".into(), + )); + } + Ok(()) +} +fn receipt_from( + row: snapshot_operation::Model, + request: &PublicationRequest, +) -> Result { + if row.instance_id != request.instance_id || row.request_digest != request.request_digest { + return Err(MegaError::Conflict( + "operation key reused with a different request".into(), + )); + } + let (Some(seq), Some(view_id), Some(outcome)) = (row.publication_seq, row.view_id, row.outcome) + else { + return Err(MegaError::Unavailable( + "incomplete publication receipt".into(), + )); + }; + if seq <= 0 { + return Err(MegaError::Unavailable("invalid publication receipt".into())); + } + validate_digest(&view_id)?; + let outcome = match outcome.as_str() { + "published" => PublicationOutcome::Published, + "no_op" => PublicationOutcome::NoOp, + _ => return Err(MegaError::Unavailable("unknown publication outcome".into())), + }; + Ok(PublicationReceipt { + publication_seq: seq, + view_id, + outcome, + }) +} + +#[cfg(test)] +mod tests; diff --git a/jupiter/src/storage/publication_storage/tests.rs b/jupiter/src/storage/publication_storage/tests.rs new file mode 100644 index 000000000..6ecca1c8d --- /dev/null +++ b/jupiter/src/storage/publication_storage/tests.rs @@ -0,0 +1,613 @@ +use std::sync::Arc; + +use callisto::{git_repo, import_refs, sea_orm_active_enums::RefTypeEnum}; +use sea_orm::{ConnectOptions, Database, PaginatorTrait}; +use tempfile::TempDir; + +use super::*; +use crate::storage::{ + git_db_storage::GitDbStorage, + snapshot_storage::{SnapshotStorage, SourceKind}, +}; + +struct Fixture { + publications: PublicationStorage, + refs: GitDbStorage, + instance: String, +} + +impl Fixture { + async fn new(base: BaseStorage) -> Self { + let source = SnapshotStorage { base: base.clone() } + .ensure_source(SourceKind::Native, 0) + .await + .unwrap(); + let conn = base.get_connection(); + let now = chrono::Utc::now().naive_utc(); + git_repo::Entity::insert(git_repo::ActiveModel { + id: Set(42), + repo_path: Set("/third-party/publication-fixture".into()), + repo_name: Set("publication-fixture".into()), + created_at: Set(now), + updated_at: Set(now), + }) + .exec(conn) + .await + .unwrap(); + for (id, name) in [(1, "refs/heads/main"), (2, "refs/heads/feature")] { + import_refs::Entity::insert(import_refs::ActiveModel { + id: Set(id), + repo_id: Set(42), + ref_name: Set(name.into()), + ref_git_id: Set("a".repeat(40)), + ref_type: Set(RefTypeEnum::Branch), + default_branch: Set(id == 1), + created_at: Set(now), + updated_at: Set(now), + }) + .exec(conn) + .await + .unwrap(); + } + Self { + publications: PublicationStorage { base: base.clone() }, + refs: GitDbStorage { base }, + instance: source.instance_id, + } + } + + fn request(&self, label: &str) -> PublicationRequest { + PublicationRequest { + instance_id: self.instance.clone(), + actor_domain: "test:actor".into(), + operation_id: uuid::Uuid::new_v4().to_string(), + request_digest: node_digest(label.as_bytes()), + } + } + fn view(&self, label: &str) -> PreparedNamespaceView { + // Opaque storage fixture only. Shared Ceres/ScorpioFS manifest tests use + // independently framed real descriptors; this facade does not decode them. + let bytes = format!("{}:view:{label}", self.instance).into_bytes(); + PreparedNamespaceView { + instance_id: self.instance.clone(), + view_id: node_digest(&bytes), + canonical_bytes: bytes, + } + } + async fn ready( + &self, + request: PublicationRequest, + head: Option, + ) -> Box { + match self.publications.begin(request, head, 1).await.unwrap() { + BeginPublication::Ready(txn) => txn, + BeginPublication::Replay(_) => panic!("unexpected replay"), + } + } + async fn ref_id(&self, id: i64) -> String { + import_refs::Entity::find_by_id(id) + .one(self.refs.base.get_connection()) + .await + .unwrap() + .unwrap() + .ref_git_id + } + async fn counts(&self) -> (u64, u64, u64) { + let conn = self.refs.base.get_connection(); + ( + namespace_publication::Entity::find() + .count(conn) + .await + .unwrap(), + namespace_outbox::Entity::find().count(conn).await.unwrap(), + snapshot_operation::Entity::find() + .count(conn) + .await + .unwrap(), + ) + } +} + +async fn sqlite() -> (TempDir, Fixture) { + let dir = TempDir::new().unwrap(); + let storage = crate::tests::test_storage(dir.path()).await; + let fixture = Fixture::new(storage.mono_storage().base.clone()).await; + (dir, fixture) +} + +async fn lifecycle(f: &Fixture) { + let request = f.request("bootstrap"); + let initial = f.view("B"); + let txn = f.ready(request.clone(), None).await; + assert!( + f.refs + .update_ref_if_unchanged( + 42, + "refs/heads/main", + &"a".repeat(40), + &"b".repeat(40), + txn.transaction() + ) + .await + .unwrap() + ); + let receipt = txn.finish(&initial, "initial import").await.unwrap(); + assert_eq!(receipt.publication_seq, 1); + assert_eq!(receipt.outcome, PublicationOutcome::Published); + assert_eq!(f.ref_id(1).await, "b".repeat(40)); + assert_eq!(f.counts().await, (1, 1, 1)); + assert_eq!( + f.publications.receipt(&request).await.unwrap(), + Some(receipt.clone()) + ); + assert!( + matches!(f.publications.begin(request.clone(), None, 1).await.unwrap(), + BeginPublication::Replay(found) if found == receipt) + ); + let changed_request = PublicationRequest { + request_digest: node_digest(b"different"), + ..request.clone() + }; + assert!(matches!( + f.publications.begin(changed_request, None, 1).await, + Err(MegaError::Conflict(_)) + )); + assert_eq!(f.counts().await, (1, 1, 1)); + + let head = f.publications.head(&f.instance).await.unwrap(); + let noop_request = f.request("non-selected branch"); + let txn = f.ready(noop_request, head.clone()).await; + assert!( + f.refs + .update_ref_if_unchanged( + 42, + "refs/heads/feature", + &"a".repeat(40), + &"c".repeat(40), + txn.transaction() + ) + .await + .unwrap() + ); + assert_eq!( + txn.finish(&initial, "non-selected branch") + .await + .unwrap() + .outcome, + PublicationOutcome::NoOp + ); + assert_eq!(f.counts().await, (1, 1, 2)); + assert_eq!(f.ref_id(2).await, "c".repeat(40)); + assert_eq!(f.publications.head(&f.instance).await.unwrap(), head); + + // Failure after ref mutation must not commit a receipt, view, head or ref. + let failed_request = f.request("wrong view instance"); + let txn = f.ready(failed_request.clone(), head.clone()).await; + assert!( + f.refs + .update_ref_if_unchanged( + 42, + "refs/heads/main", + &"b".repeat(40), + &"d".repeat(40), + txn.transaction() + ) + .await + .unwrap() + ); + let candidate = f.view("D"); + let wrong_instance = PreparedNamespaceView { + instance_id: uuid::Uuid::new_v4().to_string(), + ..candidate.clone() + }; + assert!(txn.finish(&wrong_instance, "must rollback").await.is_err()); + assert_eq!(f.ref_id(1).await, "b".repeat(40)); + assert!( + f.publications + .receipt(&failed_request) + .await + .unwrap() + .is_none() + ); + assert!( + f.publications + .view(&f.instance, &candidate.view_id) + .await + .unwrap() + .is_none() + ); + assert_eq!(f.counts().await, (1, 1, 2)); + + // Even identical-view publication must execute the writer-epoch fence. + let fenced_request = f.request("fenced noop"); + let txn = f.ready(fenced_request.clone(), head.clone()).await; + namespace_head::Entity::update_many() + .col_expr(namespace_head::Column::WriterEpoch, Expr::value(2i64)) + .filter(namespace_head::Column::InstanceId.eq(&f.instance)) + .exec(txn.transaction()) + .await + .unwrap(); + assert!(matches!( + txn.finish(&initial, "fenced noop").await, + Err(MegaError::Conflict(_)) + )); + assert!( + f.publications + .receipt(&fenced_request) + .await + .unwrap() + .is_none() + ); + assert_eq!(f.publications.head(&f.instance).await.unwrap(), head); + + // Force the publication-row insert to fail AFTER view insert and head CAS. + // This inserted collision is itself uncommitted and must also roll back. + let failed_after_cas = f.request("failure after head CAS"); + let txn = f.ready(failed_after_cas.clone(), head.clone()).await; + assert!( + f.refs + .update_ref_if_unchanged( + 42, + "refs/heads/main", + &"b".repeat(40), + &"e".repeat(40), + txn.transaction() + ) + .await + .unwrap() + ); + namespace_publication::Entity::insert(namespace_publication::ActiveModel { + instance_id: Set(f.instance.clone()), + publication_seq: Set(2), + view_id: Set(candidate.view_id.clone()), + parent_seq: Set(Some(1)), + parent_view_id: Set(Some(initial.view_id.clone())), + writer_epoch: Set(1), + actor_domain: Set(failed_after_cas.actor_domain.clone()), + operation_id: Set(failed_after_cas.operation_id.clone()), + reason: Set("injected duplicate publication row".into()), + created_at: Set(chrono::Utc::now().fixed_offset()), + }) + .exec(txn.transaction()) + .await + .unwrap(); + assert!( + txn.finish(&candidate, "must rollback after CAS") + .await + .is_err() + ); + assert_eq!(f.ref_id(1).await, "b".repeat(40)); + assert_eq!(f.publications.head(&f.instance).await.unwrap(), head); + assert!( + f.publications + .receipt(&failed_after_cas) + .await + .unwrap() + .is_none() + ); + assert!( + f.publications + .view(&f.instance, &candidate.view_id) + .await + .unwrap() + .is_none() + ); + assert_eq!(f.counts().await, (1, 1, 2)); + + // Dropping the owner cannot commit a reserved operation or its ref writes. + let dropped = f.request("dropped transaction"); + let txn = f.ready(dropped.clone(), head.clone()).await; + assert!( + f.refs + .update_ref_if_unchanged( + 42, + "refs/heads/feature", + &"c".repeat(40), + &"f".repeat(40), + txn.transaction() + ) + .await + .unwrap() + ); + drop(txn); + // Reserve the same key again: wait on the database lock, not an arbitrary sleep. + f.ready(dropped.clone(), head.clone()) + .await + .abort() + .await + .unwrap(); + assert_eq!(f.ref_id(2).await, "c".repeat(40)); + assert!(f.publications.receipt(&dropped).await.unwrap().is_none()); + + let txn = f.ready(f.request("next root"), head.clone()).await; + assert!( + !f.refs + .update_ref_if_unchanged( + 42, + "refs/heads/main", + &"a".repeat(40), + &"e".repeat(40), + txn.transaction() + ) + .await + .unwrap() + ); + assert!( + f.refs + .update_ref_if_unchanged( + 42, + "refs/heads/main", + &"b".repeat(40), + &"d".repeat(40), + txn.transaction() + ) + .await + .unwrap() + ); + assert_eq!( + txn.finish(&candidate, "advance") + .await + .unwrap() + .publication_seq, + 2 + ); + assert_eq!( + f.publications + .view(&f.instance, &initial.view_id) + .await + .unwrap(), + Some(initial.canonical_bytes) + ); + assert!(matches!( + f.publications.begin(f.request("stale head"), head, 1).await, + Err(MegaError::Conflict(_)) + )); + assert_eq!(f.counts().await, (2, 2, 3)); +} + +#[tokio::test] +async fn snapshot_publication_sqlite_ref_receipt_noop_fence_and_rollback() { + let (_dir, f) = sqlite().await; + lifecycle(&f).await; +} + +async fn duplicate_race(f: &Fixture) { + let request = f.request("duplicate bootstrap"); + let view = f.view("B"); + let results = futures::future::join_all((0..8).map(|_| async { + match f + .publications + .begin(request.clone(), None, 1) + .await + .unwrap() + { + BeginPublication::Replay(receipt) => (false, receipt), + BeginPublication::Ready(txn) => { + assert!( + f.refs + .update_ref_if_unchanged( + 42, + "refs/heads/main", + &"a".repeat(40), + &"b".repeat(40), + txn.transaction() + ) + .await + .unwrap() + ); + (true, txn.finish(&view, "duplicate race").await.unwrap()) + } + } + })) + .await; + assert_eq!(results.iter().filter(|(first, _)| *first).count(), 1); + assert!(results.iter().all(|(_, r)| r == &results[0].1)); + assert_eq!(f.counts().await, (1, 1, 1)); + assert_eq!(f.ref_id(1).await, "b".repeat(40)); +} + +#[tokio::test] +async fn snapshot_publication_sqlite_duplicate_operation_is_applied_once() { + let (_dir, f) = sqlite().await; + duplicate_race(&f).await; +} + +async fn competing_writers(f: &Fixture) { + let initial = f.view("A"); + f.ready(f.request("bootstrap"), None) + .await + .finish(&initial, "initial") + .await + .unwrap(); + let expected = f.publications.head(&f.instance).await.unwrap(); + let results = futures::future::join_all(["b", "c"].into_iter().map(|new| { + let expected = expected.clone(); + async move { + let request = f.request(new); + let txn = match f.publications.begin(request.clone(), expected, 1).await { + Ok(BeginPublication::Ready(txn)) => txn, + Err(MegaError::Conflict(_)) => return (new, false, request), + _ => panic!("unexpected begin result"), + }; + if !f + .refs + .update_ref_if_unchanged( + 42, + "refs/heads/main", + &"a".repeat(40), + &new.repeat(40), + txn.transaction(), + ) + .await + .unwrap() + { + txn.abort().await.unwrap(); + return (new, false, request); + } + let success = match txn.finish(&f.view(new), "competing writers").await { + Ok(_) => true, + Err(MegaError::Conflict(_)) => false, + other => panic!("{other:?}"), + }; + (new, success, request) + } + })) + .await; + assert_eq!(results.iter().filter(|(_, success, _)| *success).count(), 1); + for (new, success, request) in results { + assert_eq!( + f.publications.receipt(&request).await.unwrap().is_some(), + success + ); + if success { + assert_eq!(f.ref_id(1).await, new.repeat(40)); + assert_eq!( + f.publications + .head(&f.instance) + .await + .unwrap() + .unwrap() + .view_id, + f.view(new).view_id + ); + } + } + assert_eq!(f.counts().await, (2, 2, 2)); +} + +#[tokio::test] +async fn snapshot_publication_sqlite_expected_old_writers_do_not_overwrite() { + let (_dir, f) = sqlite().await; + competing_writers(&f).await; +} + +async fn postgres() -> (Fixture, ConnectOptions) { + let url = std::env::var("MEGA_SNAPSHOT_TEST_DATABASE_URL") + .expect("explicit disposable PostgreSQL URL required"); + let parsed = url::Url::parse(&url).unwrap(); + assert!(matches!(parsed.scheme(), "postgres" | "postgresql")); + assert!(matches!( + parsed.host_str(), + Some("localhost" | "127.0.0.1" | "[::1]") + )); + assert_eq!(parsed.path(), "/snapshot_test"); + let control = Database::connect( + ConnectOptions::new(url.clone()) + .max_connections(1) + .sqlx_logging(false) + .to_owned(), + ) + .await + .unwrap(); + let schema = format!("snapshot_publication_{}", uuid::Uuid::new_v4().simple()); + control + .execute_unprepared(&format!("CREATE SCHEMA {schema}")) + .await + .unwrap(); + let options = ConnectOptions::new(url) + .max_connections(12) + .sqlx_logging(false) + .set_schema_search_path(schema.clone()) + .to_owned(); + let db = Database::connect(options.clone()).await.unwrap(); + jupiter_migrate::apply_migrations(&db, false).await.unwrap(); + println!("publication PostgreSQL schema retained: {schema}"); + (Fixture::new(BaseStorage::new(Arc::new(db))).await, options) +} + +#[tokio::test] +#[ignore = "requires explicit disposable loopback MEGA_SNAPSHOT_TEST_DATABASE_URL"] +async fn snapshot_publication_postgres_lifecycle_and_reopen() { + let (f, options) = postgres().await; + lifecycle(&f).await; + let reopened = PublicationStorage { + base: BaseStorage::new(Arc::new(Database::connect(options).await.unwrap())), + }; + assert_eq!( + reopened.head(&f.instance).await.unwrap(), + f.publications.head(&f.instance).await.unwrap() + ); + let view = f.view("B"); + assert_eq!( + reopened.view(&f.instance, &view.view_id).await.unwrap(), + Some(view.canonical_bytes) + ); + // Resolve every committed result on an independent connection after the + // caller has discarded its finish response (no repeated ref mutation). + for row in snapshot_operation::Entity::find() + .all(f.refs.base.get_connection()) + .await + .unwrap() + { + let request = PublicationRequest { + instance_id: row.instance_id, + actor_domain: row.actor_domain, + operation_id: row.operation_id, + request_digest: row.request_digest, + }; + assert_eq!( + reopened.receipt(&request).await.unwrap(), + f.publications.receipt(&request).await.unwrap() + ); + } +} + +#[tokio::test] +#[ignore = "requires explicit disposable loopback MEGA_SNAPSHOT_TEST_DATABASE_URL"] +async fn snapshot_publication_postgres_concurrent_duplicate_and_expected_old() { + let (f, _) = postgres().await; + duplicate_race(&f).await; + let (f, _) = postgres().await; + competing_writers(&f).await; +} + +#[tokio::test] +#[ignore = "requires explicit disposable loopback MEGA_SNAPSHOT_TEST_DATABASE_URL"] +async fn snapshot_publication_postgres_external_epoch_change_fences_noop() { + let (f, _) = postgres().await; + let view = f.view("A"); + f.ready(f.request("bootstrap"), None) + .await + .finish(&view, "initial") + .await + .unwrap(); + let expected = f.publications.head(&f.instance).await.unwrap(); + let request = f.request("stale writer noop"); + let txn = f.ready(request.clone(), expected).await; + // The pool supplies an independent connection; this fence commits outside + // the stale writer's transaction while its operation reservation is held. + namespace_head::Entity::update_many() + .col_expr(namespace_head::Column::WriterEpoch, Expr::value(2i64)) + .filter(namespace_head::Column::InstanceId.eq(&f.instance)) + .exec(f.refs.base.get_connection()) + .await + .unwrap(); + assert!( + f.refs + .update_ref_if_unchanged( + 42, + "refs/heads/feature", + &"a".repeat(40), + &"f".repeat(40), + txn.transaction() + ) + .await + .unwrap() + ); + assert!(matches!( + txn.finish(&view, "stale writer noop").await, + Err(MegaError::Conflict(_)) + )); + assert_eq!( + f.publications + .head(&f.instance) + .await + .unwrap() + .unwrap() + .writer_epoch, + 2 + ); + assert_eq!(f.ref_id(2).await, "a".repeat(40)); + assert!(f.publications.receipt(&request).await.unwrap().is_none()); + assert_eq!(f.counts().await, (1, 1, 1)); +} diff --git a/jupiter/src/storage/snapshot_storage.rs b/jupiter/src/storage/snapshot_storage.rs new file mode 100644 index 000000000..3cc2e3b81 --- /dev/null +++ b/jupiter/src/storage/snapshot_storage.rs @@ -0,0 +1,272 @@ +//! Persistent source identity and immutable scope attestations. Registry names +//! and live refs are intentionally not foreign keys: historical proofs must +//! survive their cleanup. Publishers can use the same database transaction. + +use callisto::{snapshot_instance, snapshot_source, source_commit_scope}; +use common::errors::MegaError; +use sea_orm::{ + ColumnTrait, ConnectionTrait, DbErr, EntityTrait, QueryFilter, Set, sea_query::OnConflict, +}; +use sha2::{Digest, Sha256}; + +use super::base_storage::{BaseStorage, StorageConnector}; + +const INSTANCE_ROW: &str = "default"; + +#[derive(Clone)] +pub struct SnapshotStorage { + pub base: BaseStorage, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SourceKind { + Native, + Import, +} + +impl SourceKind { + pub fn as_str(self) -> &'static str { + match self { + Self::Native => "native", + Self::Import => "import", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ScopeProofKind { + ImportCommit, + NativeRoot, + NativeRefObserved, + NativeScopeProjection, + NativeReceivePack, + NativeMerge, +} + +impl ScopeProofKind { + fn as_str(self) -> &'static str { + match self { + Self::ImportCommit => "import_commit", + Self::NativeRoot => "native_root", + Self::NativeRefObserved => "native_ref_observed", + Self::NativeScopeProjection => "native_scope_projection", + Self::NativeReceivePack => "native_receive_pack", + Self::NativeMerge => "native_merge", + } + } +} + +/// The application must verify the actual object/scope relationship before +/// writing this record. Syntactic validation here is defense in depth only. +#[derive(Debug, Clone)] +pub struct ScopeAttestation { + pub source_id: String, + pub scope_path: String, + pub commit_oid: String, + pub root_tree_oid: String, + pub proof_kind: ScopeProofKind, + pub proof_oid: Option, +} + +impl SnapshotStorage { + pub async fn ensure_source( + &self, + kind: SourceKind, + repo_id: i64, + ) -> Result { + self.ensure_source_in(self.base.get_connection(), kind, repo_id) + .await + } + + /// Stable under retries, concurrent registration and process restarts. + /// Caller resolves/authorizes repo_id separately; native uses reserved ID 0. + pub async fn ensure_source_in( + &self, + conn: &C, + kind: SourceKind, + repo_id: i64, + ) -> Result { + if (kind == SourceKind::Native && repo_id != 0) + || (kind == SourceKind::Import && repo_id <= 0) + { + return Err(MegaError::bad_request("invalid snapshot backend identity")); + } + let now = chrono::Utc::now(); + let inserted = snapshot_instance::Entity::insert(snapshot_instance::ActiveModel { + singleton: Set(INSTANCE_ROW.into()), + instance_id: Set(uuid::Uuid::new_v4().to_string()), + created_at: Set(now), + }) + .on_conflict( + OnConflict::column(snapshot_instance::Column::Singleton) + .do_nothing() + .to_owned(), + ) + .exec(conn) + .await; + ignore_existing(inserted)?; + let instance = snapshot_instance::Entity::find_by_id(INSTANCE_ROW.to_owned()) + .one(conn) + .await? + .ok_or_else(|| MegaError::Unavailable("snapshot instance disappeared".into()))?; + let inserted = snapshot_source::Entity::insert(snapshot_source::ActiveModel { + source_id: Set(uuid::Uuid::new_v4().to_string()), + instance_id: Set(instance.instance_id.clone()), + kind: Set(kind.as_str().into()), + repo_id: Set(repo_id), + created_at: Set(now), + }) + .on_conflict( + OnConflict::columns([ + snapshot_source::Column::InstanceId, + snapshot_source::Column::Kind, + snapshot_source::Column::RepoId, + ]) + .do_nothing() + .to_owned(), + ) + .exec(conn) + .await; + ignore_existing(inserted)?; + snapshot_source::Entity::find() + .filter(snapshot_source::Column::InstanceId.eq(instance.instance_id)) + .filter(snapshot_source::Column::Kind.eq(kind.as_str())) + .filter(snapshot_source::Column::RepoId.eq(repo_id)) + .one(conn) + .await? + .ok_or_else(|| MegaError::Unavailable("snapshot source disappeared".into())) + } + + pub async fn source( + &self, + source_id: &str, + ) -> Result, MegaError> { + Ok(snapshot_source::Entity::find_by_id(source_id.to_owned()) + .one(self.base.get_connection()) + .await?) + } + + /// Insert once; same key + different root or path is never an upsert. + /// This accepts DatabaseTransaction so proof, objects and refs can commit together. + pub async fn record_scope_in( + &self, + conn: &C, + proof: &ScopeAttestation, + ) -> Result<(), MegaError> { + validate_attestation(proof)?; + let key = scope_key(&proof.scope_path); + let inserted = source_commit_scope::Entity::insert(source_commit_scope::ActiveModel { + source_id: Set(proof.source_id.clone()), + scope_key: Set(key.clone()), + scope_path: Set(proof.scope_path.clone()), + object_format: Set("sha1".into()), + commit_oid: Set(proof.commit_oid.clone()), + root_tree_oid: Set(proof.root_tree_oid.clone()), + proof_kind: Set(proof.proof_kind.as_str().into()), + proof_oid: Set(proof.proof_oid.clone()), + created_at: Set(chrono::Utc::now()), + }) + .on_conflict( + OnConflict::columns([ + source_commit_scope::Column::SourceId, + source_commit_scope::Column::ScopeKey, + source_commit_scope::Column::ObjectFormat, + source_commit_scope::Column::CommitOid, + ]) + .do_nothing() + .to_owned(), + ) + .exec(conn) + .await; + ignore_existing(inserted)?; + let saved = source_commit_scope::Entity::find_by_id(( + proof.source_id.clone(), + key, + "sha1".to_owned(), + proof.commit_oid.clone(), + )) + .one(conn) + .await? + .ok_or_else(|| MegaError::Unavailable("scope proof disappeared".into()))?; + if saved.scope_path != proof.scope_path || saved.root_tree_oid != proof.root_tree_oid { + return Err(MegaError::Conflict( + "immutable source scope proof mismatch".into(), + )); + } + Ok(()) + } + + pub async fn scope( + &self, + source_id: &str, + path: &str, + commit_oid: &str, + ) -> Result, MegaError> { + let result = source_commit_scope::Entity::find_by_id(( + source_id.to_owned(), + scope_key(path), + "sha1".to_owned(), + commit_oid.to_owned(), + )) + .one(self.base.get_connection()) + .await?; + if result + .as_ref() + .is_some_and(|proof| proof.scope_path != path) + { + return Err(MegaError::Unavailable( + "scope path digest collision or corrupt proof".into(), + )); + } + Ok(result) + } +} + +fn ignore_existing(result: Result) -> Result<(), MegaError> { + match result { + Ok(_) | Err(DbErr::RecordNotInserted) => Ok(()), + Err(error) => Err(error.into()), + } +} + +fn scope_key(path: &str) -> String { + let mut hash = Sha256::new(); + hash.update(b"mega.scope-path.v1\0"); + hash.update(path.as_bytes()); + hex::encode(hash.finalize()) +} + +fn valid_oid(oid: &str) -> bool { + oid.len() == 40 + && oid + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} + +fn validate_attestation(proof: &ScopeAttestation) -> Result<(), MegaError> { + let valid_path = proof.scope_path == "/" + || (proof.scope_path.len() <= 4096 + && proof.scope_path.strip_prefix('/').is_some_and(|relative| { + relative.split('/').all(|part| { + !part.is_empty() + && part != "." + && part != ".." + && !part.contains('\0') + && part.len() <= 255 + }) + })); + let valid_source = uuid::Uuid::parse_str(&proof.source_id) + .is_ok_and(|id| !id.is_nil() && id.to_string() == proof.source_id); + if !valid_path + || !valid_source + || !valid_oid(&proof.commit_oid) + || !valid_oid(&proof.root_tree_oid) + || proof.proof_oid.as_ref().is_some_and(|oid| !valid_oid(oid)) + { + return Err(MegaError::bad_request("invalid source scope attestation")); + } + Ok(()) +} + +#[cfg(test)] +mod tests; diff --git a/jupiter/src/storage/snapshot_storage/tests.rs b/jupiter/src/storage/snapshot_storage/tests.rs new file mode 100644 index 000000000..ae09bcfd3 --- /dev/null +++ b/jupiter/src/storage/snapshot_storage/tests.rs @@ -0,0 +1,273 @@ +use callisto::{import_refs, sea_orm_active_enums::RefTypeEnum}; +use tempfile::TempDir; + +use super::*; + +fn proof(source_id: &str, path: &str) -> ScopeAttestation { + ScopeAttestation { + source_id: source_id.into(), + scope_path: path.into(), + commit_oid: "1".repeat(40), + root_tree_oid: "2".repeat(40), + proof_kind: ScopeProofKind::NativeScopeProjection, + proof_oid: Some("3".repeat(40)), + } +} + +#[tokio::test] +async fn source_identity_is_persistent_and_backend_scoped() { + let dir = TempDir::new().unwrap(); + let storage = crate::tests::test_storage(dir.path()).await; + let snapshots = storage.snapshot_storage(); + let native = snapshots + .ensure_source(SourceKind::Native, 0) + .await + .unwrap(); + let imported = snapshots + .ensure_source(SourceKind::Import, 17) + .await + .unwrap(); + let another = snapshots + .ensure_source(SourceKind::Import, 18) + .await + .unwrap(); + assert_eq!(native.instance_id, imported.instance_id); + assert_ne!(native.source_id, imported.source_id); + assert_ne!(imported.source_id, another.source_id); + let connection = + sea_orm::Database::connect(format!("sqlite://{}", dir.path().join("test.db").display())) + .await + .unwrap(); + let reopened = SnapshotStorage { + base: BaseStorage::new(std::sync::Arc::new(connection)), + }; + assert_eq!( + reopened + .ensure_source(SourceKind::Import, 17) + .await + .unwrap() + .source_id, + imported.source_id + ); + assert_eq!( + reopened + .source(&native.source_id) + .await + .unwrap() + .unwrap() + .repo_id, + 0 + ); + assert!( + snapshots + .ensure_source(SourceKind::Native, 17) + .await + .is_err() + ); + assert!( + snapshots + .ensure_source(SourceKind::Import, 0) + .await + .is_err() + ); +} + +#[tokio::test] +async fn concurrent_registration_allocates_one_identity() { + let dir = TempDir::new().unwrap(); + let storage = crate::tests::test_storage(dir.path()).await; + let snapshots = storage.snapshot_storage(); + let results = + futures::future::join_all((0..16).map(|_| snapshots.ensure_source(SourceKind::Import, 42))) + .await; + let sources = results.into_iter().map(Result::unwrap).collect::>(); + assert!( + sources + .iter() + .all(|source| source.source_id == sources[0].source_id + && source.instance_id == sources[0].instance_id) + ); +} + +#[tokio::test] +async fn proof_allows_multiple_scopes_but_never_overwrites_a_root() { + let dir = TempDir::new().unwrap(); + let storage = crate::tests::test_storage(dir.path()).await; + let snapshots = storage.snapshot_storage(); + let source = snapshots + .ensure_source(SourceKind::Native, 0) + .await + .unwrap(); + let a = proof(&source.source_id, "/project/a"); + let b = proof(&source.source_id, "/project/b"); + let conn = snapshots.base.get_connection(); + snapshots.record_scope_in(conn, &a).await.unwrap(); + snapshots.record_scope_in(conn, &a).await.unwrap(); + snapshots.record_scope_in(conn, &b).await.unwrap(); + for path in ["/project/a", "/project/b"] { + assert_eq!( + snapshots + .scope(&source.source_id, path, &a.commit_oid) + .await + .unwrap() + .unwrap() + .root_tree_oid, + a.root_tree_oid + ); + } + let mut replacement = a.clone(); + replacement.root_tree_oid = "4".repeat(40); + assert!(matches!( + snapshots.record_scope_in(conn, &replacement).await, + Err(MegaError::Conflict(_)) + )); + assert_eq!( + snapshots + .scope(&source.source_id, &a.scope_path, &a.commit_oid) + .await + .unwrap() + .unwrap() + .root_tree_oid, + a.root_tree_oid + ); + assert!( + snapshots + .scope(&source.source_id, "/project/ab", &a.commit_oid) + .await + .unwrap() + .is_none() + ); +} + +#[tokio::test] +async fn scope_proof_participates_in_the_callers_transaction() { + let dir = TempDir::new().unwrap(); + let storage = crate::tests::test_storage(dir.path()).await; + let snapshots = storage.snapshot_storage(); + let source = snapshots + .ensure_source(SourceKind::Native, 0) + .await + .unwrap(); + let attestation = proof(&source.source_id, "/project/a"); + let txn = storage.begin_db_transaction().await.unwrap(); + snapshots.record_scope_in(&txn, &attestation).await.unwrap(); + txn.rollback().await.unwrap(); + assert!( + snapshots + .scope( + &source.source_id, + &attestation.scope_path, + &attestation.commit_oid + ) + .await + .unwrap() + .is_none() + ); + let txn = storage.begin_db_transaction().await.unwrap(); + snapshots.record_scope_in(&txn, &attestation).await.unwrap(); + txn.commit().await.unwrap(); + assert!( + snapshots + .scope( + &source.source_id, + &attestation.scope_path, + &attestation.commit_oid + ) + .await + .unwrap() + .is_some() + ); +} + +#[tokio::test] +async fn ref_cleanup_does_not_remove_history_and_fk_prevents_source_cascade() { + let dir = TempDir::new().unwrap(); + let storage = crate::tests::test_storage(dir.path()).await; + let snapshots = storage.snapshot_storage(); + let source = snapshots + .ensure_source(SourceKind::Import, 17) + .await + .unwrap(); + let mut attestation = proof(&source.source_id, "/third-party/lib"); + attestation.proof_kind = ScopeProofKind::ImportCommit; + let now = chrono::Utc::now().naive_utc(); + storage + .git_db_storage() + .save_ref( + 17, + import_refs::Model { + id: common::utils::generate_id(), + repo_id: 17, + ref_name: "refs/heads/old".into(), + ref_git_id: attestation.commit_oid.clone(), + ref_type: RefTypeEnum::Branch, + default_branch: false, + created_at: now, + updated_at: now, + }, + ) + .await + .unwrap(); + snapshots + .record_scope_in(snapshots.base.get_connection(), &attestation) + .await + .unwrap(); + storage + .git_db_storage() + .remove_ref(17, "refs/heads/old") + .await + .unwrap(); + assert!( + snapshots + .scope( + &source.source_id, + &attestation.scope_path, + &attestation.commit_oid + ) + .await + .unwrap() + .is_some() + ); + assert!( + snapshot_source::Entity::delete_by_id(source.source_id.clone()) + .exec(snapshots.base.get_connection()) + .await + .is_err() + ); + assert!(snapshots.source(&source.source_id).await.unwrap().is_some()); +} + +#[tokio::test] +async fn long_paths_use_bounded_index_keys_and_bad_paths_are_rejected() { + let dir = TempDir::new().unwrap(); + let storage = crate::tests::test_storage(dir.path()).await; + let snapshots = storage.snapshot_storage(); + let source = snapshots + .ensure_source(SourceKind::Native, 0) + .await + .unwrap(); + let path = format!("/{}", vec!["x".repeat(250); 15].join("/")); + let attestation = proof(&source.source_id, &path); + snapshots + .record_scope_in(snapshots.base.get_connection(), &attestation) + .await + .unwrap(); + let saved = snapshots + .scope(&source.source_id, &path, &attestation.commit_oid) + .await + .unwrap() + .unwrap(); + assert_eq!(saved.scope_key.len(), 64); + assert_eq!(saved.scope_path, path); + for invalid in ["relative", "/a/../b", "/a/", "/a//b", "/a\0b"] { + assert!( + snapshots + .record_scope_in( + snapshots.base.get_connection(), + &proof(&source.source_id, invalid) + ) + .await + .is_err() + ); + } +}