From 9e25570a60ab0db903fb83d43da33dee9d6e0cab Mon Sep 17 00:00:00 2001 From: nol4lej Date: Thu, 30 Jul 2026 17:09:48 -0400 Subject: [PATCH 1/2] feat(shielded-pool): multi-tree forest - seal full trees, roll over to new ones --- Cargo.lock | 2 +- client/rpc-v2/src/privacy.rs | 35 ++- .../evm/precompile/shielded-pool/src/mock.rs | 2 + frame/shielded-pool/CHANGELOG.md | 52 ++++ frame/shielded-pool/Cargo.toml | 2 +- frame/shielded-pool/runtime-api/src/lib.rs | 10 + frame/shielded-pool/src/lib.rs | 78 +++++- frame/shielded-pool/src/merkle.rs | 256 ++++++++++++++++-- frame/shielded-pool/src/migrations.rs | 34 ++- frame/shielded-pool/src/mock.rs | 2 + frame/shielded-pool/src/runtime_api_impl.rs | 72 +++++ frame/shielded-pool/src/storage.rs | 13 +- template/runtime/src/lib.rs | 15 +- ts-tests/test-forest-e2e.ts | 106 ++++++++ 14 files changed, 644 insertions(+), 35 deletions(-) create mode 100644 ts-tests/test-forest-e2e.ts diff --git a/Cargo.lock b/Cargo.lock index a7e0b856..a6dd65ce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8174,7 +8174,7 @@ dependencies = [ [[package]] name = "pallet-shielded-pool" -version = "0.11.0" +version = "0.12.0" dependencies = [ "ark-bn254", "ark-ff 0.5.0", diff --git a/client/rpc-v2/src/privacy.rs b/client/rpc-v2/src/privacy.rs index f5c2586f..d0c8005d 100644 --- a/client/rpc-v2/src/privacy.rs +++ b/client/rpc-v2/src/privacy.rs @@ -69,7 +69,9 @@ fn map_key(item: &[u8], k: &[u8]) -> Vec { /// Response for `privacy_getMerkleProof`. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct MerkleProofResponse { - /// Current Merkle root read from the same block as the proof (`0x`-prefixed hex). + /// Root the leaf's tree anchors to, read at the same block as the proof + /// (`0x`-prefixed hex): the permanent sealed root for completed trees, + /// the live root for the active tree. pub root: String, /// Sibling path — 20 `0x`-prefixed hex strings. pub path: Vec, @@ -77,6 +79,8 @@ pub struct MerkleProofResponse { pub leaf_index: u32, /// Tree depth (always 20 for circuit compatibility). pub tree_depth: u32, + /// Forest tree the leaf belongs to (0 until the first tree seals). + pub tree_id: u32, } /// Response for `privacy_getNullifierStatus`. @@ -237,7 +241,7 @@ where let best_hash = self.client.info().best_hash; let api = self.client.runtime_api(); - let (root, tree_size, tree_depth) = api + let (_, tree_size, tree_depth) = api .get_merkle_tree_info(best_hash) .map_err(internal_error)?; @@ -255,6 +259,16 @@ where .map_err(internal_error)? .ok_or_else(|| internal_error(format!("no proof for leaf_index {leaf_index}")))?; + // Sealed trees anchor to their permanent root, not the active one. + // A missing entry for an existing leaf means broken sealed-root state; + // erroring beats silently serving the wrong anchor. + let (root, tree_id) = api + .get_root_for_leaf(best_hash, leaf_index) + .map_err(internal_error)? + .ok_or_else(|| { + internal_error(format!("no anchoring root for leaf_index {leaf_index}")) + })?; + Ok(MerkleProofResponse { root: format!("0x{}", hex::encode(root)), path: proof @@ -264,6 +278,7 @@ where .collect(), leaf_index, tree_depth, + tree_id, }) } @@ -282,7 +297,7 @@ where } let target = H256::from_slice(&bytes); - let (root, tree_size, tree_depth) = api + let (_, tree_size, tree_depth) = api .get_merkle_tree_info(best_hash) .map_err(internal_error)?; @@ -299,6 +314,16 @@ where )) })?; + // Sealed trees anchor to their permanent root, not the active one. + // A missing entry for an existing leaf means broken sealed-root state; + // erroring beats silently serving the wrong anchor. + let (root, tree_id) = api + .get_root_for_leaf(best_hash, leaf_index) + .map_err(internal_error)? + .ok_or_else(|| { + internal_error(format!("no anchoring root for leaf_index {leaf_index}")) + })?; + Ok(MerkleProofResponse { root: format!("0x{}", hex::encode(root)), path: proof @@ -308,6 +333,7 @@ where .collect(), leaf_index, tree_depth, + tree_id, }) } @@ -492,11 +518,13 @@ mod tests { path: vec!["0x1111".to_string(), "0x2222".to_string()], leaf_index: 7, tree_depth: 20, + tree_id: 0, }; let json = serde_json::to_value(&resp).unwrap(); assert_eq!(json["root"], "0xaabb"); assert_eq!(json["leaf_index"], 7); assert_eq!(json["tree_depth"], 20); + assert_eq!(json["tree_id"], 0); assert_eq!(json["path"].as_array().unwrap().len(), 2); } @@ -507,6 +535,7 @@ mod tests { path: vec!["0xaa".to_string()], leaf_index: 3, tree_depth: 20, + tree_id: 0, }; let back: MerkleProofResponse = serde_json::from_str(&serde_json::to_string(&orig).unwrap()).unwrap(); diff --git a/frame/evm/precompile/shielded-pool/src/mock.rs b/frame/evm/precompile/shielded-pool/src/mock.rs index a3c7170e..86cac396 100644 --- a/frame/evm/precompile/shielded-pool/src/mock.rs +++ b/frame/evm/precompile/shielded-pool/src/mock.rs @@ -120,6 +120,7 @@ parameter_types! { pub const ShieldedPoolPalletId: PalletId = PalletId(*b"shldpool"); pub const MaxTreeDepth: u32 = 20; pub const MaxHistoricRoots: u32 = 100; + pub const MaxLeavesPerTree: u32 = 8; pub const MinShieldAmount: u128 = 100; } @@ -256,6 +257,7 @@ impl pallet_shielded_pool::Config for Test { type PalletId = ShieldedPoolPalletId; type MaxTreeDepth = MaxTreeDepth; type MaxHistoricRoots = MaxHistoricRoots; + type MaxLeavesPerTree = MaxLeavesPerTree; type MinShieldAmount = MinShieldAmount; type WeightInfo = (); type Relayer = MockRelayer; diff --git a/frame/shielded-pool/CHANGELOG.md b/frame/shielded-pool/CHANGELOG.md index 165b4c5b..1b6de227 100644 --- a/frame/shielded-pool/CHANGELOG.md +++ b/frame/shielded-pool/CHANGELOG.md @@ -2,6 +2,58 @@ All notable changes to `pallet-shielded-pool` will be documented in this file. +## [0.12.0] - 2026-07-30 + +### Added +- **Multi-tree forest: the 2^20-note network ceiling is gone.** When a tree + reaches `MaxLeavesPerTree` the filling insert seals it and inserts continue + in a fresh tree. The global u32 leaf index never resets + (`tree_id = leaf_index / MaxLeavesPerTree`), so event shapes, indexer + chunking and wallet scan cursors are untouched. Zero circuit, VK, + extrinsic-signature or precompile-ABI changes — anchoring stays root-only. +- New storage `SealedTreeRoots` / `SealedRootIndex`: a sealed tree's final + root is a **permanent** anchor (bounded by tree count, max 4096 — never + evicted, unlike the historic ring), so notes in sealed trees stay spendable + forever. `MerkleRepository::is_known_root` = historic ring OR sealed set; + transfer/unshield/validate_unsigned call sites unchanged. Regression test: + a sealed root survives `MaxHistoricRoots + N` later inserts. +- New Config `MaxLeavesPerTree` (production 2^20; `integrity_test` enforces + power-of-two ≤ 2^`MAX_TREE_DEPTH` — clients derive `tree_id` from this + constant, so it must never change on a live chain). Mocks use 8 to make + rollover testable. +- New event `TreeSealed { tree_id, final_root, first_leaf_index, leaf_count }`, + emitted after the `MerkleRootUpdated` that carries the final root. +- Runtime API v2 (`api_version(2)`): `get_forest_info()` and + `get_root_for_leaf(leaf_index)` — sealed trees resolve to their permanent + root, the active tree to the live `PoseidonRoot`. +- `STORAGE_VERSION` 2 with version-only `migrations::v2::MigrateToV2` + (sealed maps start empty). Forest invariants merged into the existing + `try_state` hook: active root always known, one sealed root per completed + tree, sealed maps bijective. + +### Changed +- `Error::MerkleTreeFull` now means the absolute forest ceiling (u32 + leaf-index space, ~4096 trees) — practically unreachable — instead of the + per-tree 2^20 cap. +- `insert_leaf` / `get_merkle_path` index `MerkleNodes` by tree-local + position (`leaf_index % MaxLeavesPerTree`). Identical to the previous + global indexing for tree 0, so v1-backfilled data needs no migration. +- Consensus-affecting (seal writes new storage on the filling insert): + requires a runtime `spec_version` bump at release. `transaction_version` + unchanged. + +### Fixed +- Privacy RPC (`fc-rpc-v2`): proof endpoints no longer fall back to the + active root when a leaf's anchoring root cannot be resolved — that would + have served a sealed-tree path with the wrong anchor; they now return an + explicit error. Both endpoints gain a `tree_id` response field (additive). + +### Verified E2E +- Against a local dev node with an 8-leaf cap: 10 shields cross the first + seal; `TreeSealed` fires; sealed leaves anchor to the permanent final root + and active leaves to the live root via `privacy_getMerkleProof*` + (`ts-tests/test-forest-e2e.ts`). + ## [0.11.0] - 2026-07-30 ### Added diff --git a/frame/shielded-pool/Cargo.toml b/frame/shielded-pool/Cargo.toml index 9fad19d6..a09cd33a 100644 --- a/frame/shielded-pool/Cargo.toml +++ b/frame/shielded-pool/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pallet-shielded-pool" -version = "0.11.0" +version = "0.12.0" description = "Shielded pool pallet for private transactions using ZK proofs" authors = ["Orbinum Team"] license = "GPL-3.0-or-later" diff --git a/frame/shielded-pool/runtime-api/src/lib.rs b/frame/shielded-pool/runtime-api/src/lib.rs index bc7a5ec6..367594d6 100644 --- a/frame/shielded-pool/runtime-api/src/lib.rs +++ b/frame/shielded-pool/runtime-api/src/lib.rs @@ -26,6 +26,7 @@ pub struct RelayConfig { } sp_api::decl_runtime_apis! { + #[api_version(2)] pub trait ShieldedPoolRuntimeApi { /// Get the Merkle tree information (root, size, depth) fn get_merkle_tree_info() -> (Hash, u32, u32); @@ -37,6 +38,15 @@ sp_api::decl_runtime_apis! { /// O(1) index lookup plus O(depth) sibling reads from stored nodes. fn get_merkle_proof_for_commitment(commitment: Hash) -> Option<(u32, DefaultMerklePath)>; + /// Forest summary: (active_root, global_size, depth, current_tree_id, + /// sealed_tree_count). + fn get_forest_info() -> (Hash, u32, u32, u32, u32); + + /// Root the given leaf's tree anchors to, plus its tree_id: the + /// permanent sealed root for completed trees, the live PoseidonRoot + /// for the active tree. None if the leaf does not exist. + fn get_root_for_leaf(leaf_index: u32) -> Option<(Hash, u32)>; + /// Return relay configuration sourced from the runtime. /// /// Called by the node-native EVM relay on every `relay_shielded_call` to obtain diff --git a/frame/shielded-pool/src/lib.rs b/frame/shielded-pool/src/lib.rs index 7640a884..a346b3b6 100644 --- a/frame/shielded-pool/src/lib.rs +++ b/frame/shielded-pool/src/lib.rs @@ -102,9 +102,12 @@ pub mod pallet { pub type BalanceOf = <::Currency as Currency<::AccountId>>::Balance; - /// Storage version. v1 adds `MerkleNodes` (internal Merkle tree nodes), - /// backfilled from `MerkleLeaves` by `migrations::v1::MigrateToV1`. - pub const STORAGE_VERSION: StorageVersion = StorageVersion::new(1); + /// Storage version history: + /// - v1: `MerkleNodes` (internal Merkle tree nodes), backfilled by + /// `migrations::v1::MigrateToV1`. + /// - v2: multi-tree forest — `SealedTreeRoots` / `SealedRootIndex` + /// (start empty; version-only bump in `migrations::v2::MigrateToV2`). + pub const STORAGE_VERSION: StorageVersion = StorageVersion::new(2); #[pallet::pallet] #[pallet::storage_version(STORAGE_VERSION)] @@ -133,6 +136,13 @@ pub mod pallet { #[pallet::constant] type MaxTreeDepth: Get; + /// Leaves per tree before it seals and the forest rolls over to a new + /// tree. Must be a power of two ≤ 2^MaxTreeDepth. Production pins + /// 2^20; changing it on a live chain would re-map every note's + /// tree_id and is forbidden (see `integrity_test`). + #[pallet::constant] + type MaxLeavesPerTree: Get; + /// Maximum number of historic roots to keep #[pallet::constant] type MaxHistoricRoots: Get; @@ -207,6 +217,18 @@ pub mod pallet { pub type NullifierSet = StorageMap<_, Blake2_128Concat, Nullifier, BlockNumberFor, OptionQuery>; + /// Final root of each sealed tree, keyed by tree_id. Permanent — a sealed + /// tree is immutable forever, so its root must never expire or every note + /// still inside it would become unspendable. Bounded by tree count + /// (max 4096), not by activity. + #[pallet::storage] + pub type SealedTreeRoots = StorageMap<_, Twox64Concat, u32, Hash, OptionQuery>; + + /// Reverse index of `SealedTreeRoots`: sealed root -> tree_id. Gives + /// `is_known_root` an O(1) membership check alongside the historic ring. + #[pallet::storage] + pub type SealedRootIndex = StorageMap<_, Blake2_128Concat, Hash, u32, OptionQuery>; + /// Historic Poseidon Merkle roots (for proving against recent states) #[pallet::storage] pub type HistoricPoseidonRoots = StorageMap<_, Blake2_128Concat, Hash, bool, ValueQuery>; @@ -321,6 +343,14 @@ pub mod pallet { "MaxHistoricRoots must be non-zero, otherwise the historic-root map \ grows unbounded and the root window never evicts" ); + + let cap = T::MaxLeavesPerTree::get(); + assert!( + cap.is_power_of_two() && cap <= (1u32 << crate::types::MAX_TREE_DEPTH), + "MaxLeavesPerTree must be a power of two <= 2^MAX_TREE_DEPTH; \ + clients derive tree_id from the global leaf index using this \ + constant, so it must never change on a live chain" + ); } /// Ledger-solvency invariant: the tracked native-asset pool balance must @@ -339,6 +369,26 @@ pub mod pallet { "shielded-pool native ledger drifted from physical pool balance" ) ); + + // Forest invariants: the active root must always be provable + // against; one sealed root per completed tree; the sealed maps + // are a bijection. + use crate::storage::MerkleRepository; + frame_support::ensure!( + MerkleRepository::is_known_root::(&MerkleRepository::get_poseidon_root::()), + sp_runtime::TryRuntimeError::Other("PoseidonRoot not in known-roots set") + ); + let sealed = SealedTreeRoots::::iter().count() as u32; + frame_support::ensure!( + sealed == MerkleRepository::get_tree_size::() / T::MaxLeavesPerTree::get(), + sp_runtime::TryRuntimeError::Other("sealed-tree count != tree_size / cap") + ); + for (tree_id, root) in SealedTreeRoots::::iter() { + frame_support::ensure!( + SealedRootIndex::::get(root) == Some(tree_id), + sp_runtime::TryRuntimeError::Other("SealedRootIndex out of sync") + ); + } Ok(()) } } @@ -404,10 +454,27 @@ pub mod pallet { old_root: Hash, /// New root new_root: Hash, - /// New tree size + /// Total leaves ever inserted, global across the whole forest — + /// never per-tree. Indexer chunking and wallet scan cursors rely + /// on this being dense and monotonic; per-tree size is derivable + /// as `tree_size % MaxLeavesPerTree`. tree_size: u32, }, + /// A tree reached `MaxLeavesPerTree` and was sealed; inserts continue + /// in a fresh tree. The final root stays valid forever via + /// `SealedTreeRoots`. + TreeSealed { + /// Id of the sealed tree (global_leaf_index >> log2(MaxLeavesPerTree)) + tree_id: u32, + /// Final root of the sealed tree — permanently spendable anchor + final_root: Hash, + /// Global index of the sealed tree's first leaf + first_leaf_index: u32, + /// Leaves in the sealed tree (always MaxLeavesPerTree) + leaf_count: u32, + }, + /// Asset was registered in the registry AssetRegistered { /// The asset ID @@ -453,7 +520,8 @@ pub mod pallet { NullifierAlreadyUsed, /// The Merkle root is not recognized UnknownMerkleRoot, - /// The Merkle tree is full + /// Absolute forest capacity reached (u32 leaf-index space exhausted: + /// 4096 trees × MaxLeavesPerTree). Practically unreachable. MerkleTreeFull, /// The ZK proof is invalid InvalidProof, diff --git a/frame/shielded-pool/src/merkle.rs b/frame/shielded-pool/src/merkle.rs index cba0963a..8115bc0c 100644 --- a/frame/shielded-pool/src/merkle.rs +++ b/frame/shielded-pool/src/merkle.rs @@ -314,21 +314,24 @@ impl MerkleTreeService { /// replacing the former O(n) full recomputation from all leaves. pub fn insert_leaf(commitment: Commitment) -> Result { let index = MerkleRepository::get_tree_size::(); - let max_leaves = 2u32.saturating_pow(crate::types::MAX_TREE_DEPTH); - ensure!(index < max_leaves, Error::::MerkleTreeFull); + // Absolute forest ceiling: the global u32 leaf index must stay + // representable (4096 trees at depth 20). Per-tree fullness rolls + // over to a fresh tree below instead of erroring. + ensure!(index < u32::MAX, Error::::MerkleTreeFull); ensure!( !CommitmentMemos::::contains_key(commitment), Error::::CommitmentAlreadyExists ); + let cap = T::MaxLeavesPerTree::get(); + let tree_id = index / cap; + let local = index % cap; + // Load frontier from storage and run one incremental update. // Depth is always DEFAULT_TREE_DEPTH (20) — matches the fixed-size frontier array. let mut frontier = MerkleRepository::get_frontier::(); let mut current_hash = commitment.0; - let mut current_index = index; - // tree_id is 0 while the pool runs a single tree (index < 2^MAX_TREE_DEPTH - // is enforced above); MerkleNodes is keyed by tree_id for forest readiness. - let tree_id = index >> crate::types::MAX_TREE_DEPTH; + let mut current_index = local; for (level, frontier_slot) in frontier.iter_mut().enumerate() { if current_index % 2 == 0 { @@ -364,14 +367,42 @@ impl MerkleTreeService { MerkleRepository::set_poseidon_root::(new_poseidon_root); Self::add_poseidon_historic_root::(new_poseidon_root); + // The freshly inserted leaf belongs to `new_poseidon_root`, so this + // event fires before any seal resets the active root. Pallet::::deposit_event(Event::MerkleRootUpdated { old_root: old_poseidon_root, new_root: new_poseidon_root, tree_size: index.saturating_add(1), }); + + if local + 1 == cap { + Self::seal_tree::(tree_id, new_poseidon_root, cap); + } Ok(index) } + /// Seal a full tree and open a fresh one, eagerly in the same insert. + /// + /// The final root becomes a permanent anchor (`SealedTreeRoots` / + /// `SealedRootIndex`) — unlike the historic ring it never expires, so + /// notes in sealed trees stay spendable forever. The active tree resets + /// to the empty state; the empty root joins the historic ring to keep + /// the `PoseidonRoot ∈ known roots` invariant. + fn seal_tree(tree_id: u32, final_root: Hash, cap: u32) { + MerkleRepository::insert_sealed_root::(tree_id, final_root); + MerkleRepository::set_frontier::([[0u8; 32]; crate::types::DEFAULT_TREE_DEPTH]); + let empty_root = get_zero_hash_cached(crate::types::DEFAULT_TREE_DEPTH); + MerkleRepository::set_poseidon_root::(empty_root); + Self::add_poseidon_historic_root::(empty_root); + + Pallet::::deposit_event(Event::TreeSealed { + tree_id, + final_root, + first_leaf_index: tree_id.saturating_mul(cap), + leaf_count: cap, + }); + } + pub(crate) fn add_poseidon_historic_root(poseidon_root: Hash) { let mut order = MerkleRepository::get_historic_roots_order::(); if order.len() >= T::MaxHistoricRoots::get() as usize { @@ -404,16 +435,20 @@ impl MerkleTreeService { } let depth = crate::types::DEFAULT_TREE_DEPTH; - let tree_id = leaf_index >> crate::types::MAX_TREE_DEPTH; + let cap = T::MaxLeavesPerTree::get(); + let tree_id = leaf_index / cap; + let local = leaf_index % cap; let mut siblings = [[0u8; 32]; crate::types::DEFAULT_TREE_DEPTH]; let mut indices = [0u8; crate::types::DEFAULT_TREE_DEPTH]; for level in 0..depth { - let node_index = leaf_index >> level; + let node_index = local >> level; indices[level] = (node_index & 1) as u8; let sibling_index = node_index ^ 1; let sibling = if level == 0 { - MerkleRepository::get_leaf::(sibling_index).map(|c| c.0) + // Level-0 nodes are the leaves; map the tree-local sibling + // back to its global MerkleLeaves index. + MerkleRepository::get_leaf::(tree_id * cap + sibling_index).map(|c| c.0) } else { MerkleRepository::get_node::(tree_id, level as u8, sibling_index) }; @@ -823,8 +858,9 @@ mod tests { #[test] fn stored_node_paths_match_recomputed_reference_for_every_leaf() { new_test_ext().execute_with(|| { - // 37 leaves: odd count exercises zero-hash padding at several levels. - let leaves: Vec<[u8; 32]> = (0..37u8).map(|i| [i + 1; 32]).collect(); + // 7 leaves (< MaxLeavesPerTree): odd count exercises zero-hash + // padding without sealing the tree. + let leaves: Vec<[u8; 32]> = (0..7u8).map(|i| [i + 1; 32]).collect(); for leaf in &leaves { MerkleTreeService::insert_leaf::(Commitment::new(*leaf)).unwrap(); } @@ -848,12 +884,12 @@ mod tests { #[test] fn first_and_last_leaf_paths_verify() { new_test_ext().execute_with(|| { - let leaves: Vec<[u8; 32]> = (0..8u8).map(|i| [0xA0 + i; 32]).collect(); + let leaves: Vec<[u8; 32]> = (0..6u8).map(|i| [0xA0 + i; 32]).collect(); for leaf in &leaves { MerkleTreeService::insert_leaf::(Commitment::new(*leaf)).unwrap(); } let root = crate::storage::MerkleRepository::get_poseidon_root::(); - for i in [0u32, 7] { + for i in [0u32, 5] { let path = MerkleTreeService::get_merkle_path::(i).unwrap(); assert!(MerkleTreeService::verify_merkle_proof( &root, @@ -1037,15 +1073,197 @@ mod tests { }); } - /// The capacity guard is bound by the real tree depth, not the config, so it - /// fires at exactly 2^MAX_TREE_DEPTH regardless of MaxTreeDepth. + /// The per-tree capacity must divide the fixed depth-20 leaf space so the + /// forest's global u32 index spans whole trees. #[test] - fn capacity_guard_uses_fixed_depth() { + fn per_tree_capacity_fits_fixed_depth() { use crate::types::MAX_TREE_DEPTH; assert_eq!(MAX_TREE_DEPTH, 20); - // insert_leaf's max_leaves is 2^MAX_TREE_DEPTH; confirm the constant the - // guard reads matches the frontier depth (20 levels). - assert_eq!(2u32.saturating_pow(MAX_TREE_DEPTH), 1 << 20); + let cap = ::MaxLeavesPerTree::get(); + assert!(cap.is_power_of_two() && cap <= 1 << MAX_TREE_DEPTH); + } + + // ── Multi-tree forest: sealing and rollover ────────────────────────────── + + fn fill_leaves(from: u8, count: u8) { + for i in 0..count { + MerkleTreeService::insert_leaf::(Commitment::new([from + i; 32])).unwrap(); + } + } + + #[test] + fn filling_insert_seals_tree_and_resets_active_state() { + use crate::storage::MerkleRepository; + new_test_ext().execute_with(|| { + frame_system::Pallet::::set_block_number(1); + fill_leaves(1, 7); + let last = MerkleTreeService::insert_leaf::(Commitment::new([8u8; 32])).unwrap(); + assert_eq!(last, 7, "filling insert still returns its global index"); + + let sealed = MerkleRepository::get_sealed_root::(0).expect("tree 0 sealed"); + assert!(MerkleRepository::is_known_root::(&sealed)); + // Active tree reset: empty frontier, empty root, empty root known. + assert_eq!(MerkleRepository::get_frontier::(), [[0u8; 32]; 20]); + let empty_root = get_zero_hash_cached(20); + assert_eq!(MerkleRepository::get_poseidon_root::(), empty_root); + assert!(MerkleRepository::is_known_root::(&empty_root)); + + // Event order: MerkleRootUpdated carries the FINAL root (the new + // leaf belongs to it), then TreeSealed. + let events: sp_std::vec::Vec<_> = frame_system::Pallet::::events() + .into_iter() + .map(|r| r.event) + .collect(); + let root_pos = events + .iter() + .position(|e| { + matches!(e, crate::mock::RuntimeEvent::ShieldedPool( + Event::MerkleRootUpdated { new_root, tree_size: 8, .. } + ) if *new_root == sealed) + }) + .expect("MerkleRootUpdated with final root"); + let seal_pos = events + .iter() + .position(|e| { + matches!(e, crate::mock::RuntimeEvent::ShieldedPool( + Event::TreeSealed { tree_id: 0, final_root, first_leaf_index: 0, leaf_count: 8 } + ) if *final_root == sealed) + }) + .expect("TreeSealed event"); + assert!(root_pos < seal_pos); + }); + } + + /// The single most important forest test: a sealed tree's final root must + /// survive unbounded activity in later trees — eviction would freeze the + /// funds of every unspent note in the sealed tree. + #[test] + fn sealed_root_survives_historic_ring_eviction() { + use crate::storage::MerkleRepository; + new_test_ext().execute_with(|| { + fill_leaves(1, 8); // seal tree 0 + let sealed = MerkleRepository::get_sealed_root::(0).unwrap(); + let leaf0 = MerkleRepository::get_leaf::(0).unwrap().0; + let path0 = MerkleTreeService::get_merkle_path::(0).unwrap(); + + // MaxHistoricRoots = 100: push far past the window (also sealing + // more trees along the way). + for i in 0..120u32 { + let mut leaf = [0u8; 32]; + leaf[..4].copy_from_slice(&i.to_le_bytes()); + leaf[31] = 0xAA; + MerkleTreeService::insert_leaf::(Commitment::new(leaf)).unwrap(); + } + + assert!( + MerkleTreeService::is_known_root::(&sealed), + "sealed root must never expire" + ); + assert!( + MerkleTreeService::verify_merkle_proof(&sealed, &leaf0, &path0), + "tree-0 note must still prove against its sealed root" + ); + }); + } + + #[test] + fn straddling_inserts_land_in_consecutive_trees() { + use crate::storage::MerkleRepository; + new_test_ext().execute_with(|| { + fill_leaves(1, 7); + let a = MerkleTreeService::insert_leaf::(Commitment::new([0xE1; 32])).unwrap(); + let b = MerkleTreeService::insert_leaf::(Commitment::new([0xE2; 32])).unwrap(); + assert_eq!( + (a, b), + (7, 8), + "global index keeps counting across the seal" + ); + + // b is local leaf 0 of tree 1: its root evolved from the empty tree. + let root = MerkleRepository::get_poseidon_root::(); + let path_b = MerkleTreeService::get_merkle_path::(8).unwrap(); + assert!(MerkleTreeService::verify_merkle_proof( + &root, + &[0xE2; 32], + &path_b + )); + assert_eq!(path_b.indices, [0u8; 20], "local index 0 is all left turns"); + + // a still proves against tree 0's sealed root. + let sealed = MerkleRepository::get_sealed_root::(0).unwrap(); + let path_a = MerkleTreeService::get_merkle_path::(7).unwrap(); + assert!(MerkleTreeService::verify_merkle_proof( + &sealed, + &[0xE1; 32], + &path_a + )); + }); + } + + #[test] + fn duplicate_commitment_rejected_across_trees() { + new_test_ext().execute_with(|| { + let dup = Commitment::new([0xD7; 32]); + MerkleTreeService::insert_leaf::(dup).unwrap(); + use crate::storage::CommitmentRepository; + use crate::types::{EncryptedMemo, MAX_ENCRYPTED_MEMO_SIZE}; + CommitmentRepository::store_memo::( + dup, + EncryptedMemo::from_bytes(&[0x01u8; MAX_ENCRYPTED_MEMO_SIZE as usize]).unwrap(), + ); + fill_leaves(1, 7); // seals tree 0; now in tree 1 + assert!( + MerkleTreeService::insert_leaf::(dup).is_err(), + "same commitment in a later tree would alias the nullifier" + ); + }); + } + + /// try_state invariants must hold before, across, and after a seal. + /// Runs only with `--features try-runtime` (the hook is feature-gated). + #[cfg(feature = "try-runtime")] + #[test] + fn try_state_holds_across_seal() { + use frame_support::traits::Hooks; + new_test_ext().execute_with(|| { + let try_state = || { + as Hooks< + frame_system::pallet_prelude::BlockNumberFor, + >>::try_state(0) + }; + assert!(try_state().is_ok(), "empty forest"); + fill_leaves(1, 7); + assert!(try_state().is_ok(), "partially filled tree 0"); + fill_leaves(8, 2); // seals tree 0, opens tree 1 + assert!(try_state().is_ok(), "across the seal"); + }); + } + + #[test] + fn multiple_rollovers_keep_every_tree_provable() { + use crate::storage::MerkleRepository; + new_test_ext().execute_with(|| { + // Fill trees 0 and 1, half-fill tree 2 (cap = 8). + for i in 0..20u8 { + MerkleTreeService::insert_leaf::(Commitment::new([i + 1; 32])).unwrap(); + } + let roots = [ + MerkleRepository::get_sealed_root::(0).expect("tree 0 sealed"), + MerkleRepository::get_sealed_root::(1).expect("tree 1 sealed"), + MerkleRepository::get_poseidon_root::(), + ]; + assert!(MerkleRepository::get_sealed_root::(2).is_none()); + + for i in 0..20u32 { + let leaf = [(i + 1) as u8; 32]; + let path = MerkleTreeService::get_merkle_path::(i).unwrap(); + let root = roots[(i / 8) as usize]; + assert!( + MerkleTreeService::verify_merkle_proof(&root, &leaf, &path), + "leaf {i} must prove against its tree's root" + ); + } + }); } // ── historic-root window ────────────────────────────────────────────────── diff --git a/frame/shielded-pool/src/migrations.rs b/frame/shielded-pool/src/migrations.rs index 369db960..2fad8665 100644 --- a/frame/shielded-pool/src/migrations.rs +++ b/frame/shielded-pool/src/migrations.rs @@ -98,6 +98,34 @@ pub mod v1 { } } +pub mod v2 { + use super::*; + + /// Storage v1 -> v2: multi-tree forest. `SealedTreeRoots` and + /// `SealedRootIndex` start empty (tree 0 has never filled), so this is a + /// version-only bump — the rollover logic activates with the runtime code. + pub struct MigrateToV2(core::marker::PhantomData); + + impl OnRuntimeUpgrade for MigrateToV2 { + fn on_runtime_upgrade() -> Weight { + if Pallet::::on_chain_storage_version() >= 2 { + return T::DbWeight::get().reads(1); + } + StorageVersion::new(2).put::>(); + T::DbWeight::get().reads_writes(1, 1) + } + + #[cfg(feature = "try-runtime")] + fn post_upgrade(_state: Vec) -> Result<(), sp_runtime::TryRuntimeError> { + frame_support::ensure!( + Pallet::::on_chain_storage_version() >= 2, + sp_runtime::TryRuntimeError::Other("storage version not bumped to 2") + ); + Ok(()) + } + } +} + #[cfg(test)] mod tests { use super::v1::MigrateToV1; @@ -119,7 +147,9 @@ mod tests { #[test] fn backfill_rebuilds_nodes_and_proofs_verify() { new_test_ext().execute_with(|| { - insert_leaves(37); + // A v0-era chain only ever had a single tree; stay below the mock + // per-tree cap so the backfill precondition holds. + insert_leaves(7); let root = MerkleRepository::get_poseidon_root::(); // Simulate a v0 chain: leaves exist but internal nodes were never stored. @@ -129,7 +159,7 @@ mod tests { MigrateToV1::::on_runtime_upgrade(); assert_eq!(Pallet::::on_chain_storage_version(), 1); - for i in 0..37u32 { + for i in 0..7u32 { let leaf = MerkleRepository::get_leaf::(i).unwrap(); let path = MerkleTreeService::get_merkle_path::(i).unwrap(); assert!( diff --git a/frame/shielded-pool/src/mock.rs b/frame/shielded-pool/src/mock.rs index b1c3da0f..0f416646 100644 --- a/frame/shielded-pool/src/mock.rs +++ b/frame/shielded-pool/src/mock.rs @@ -47,6 +47,7 @@ parameter_types! { pub const ShieldedPoolPalletId: PalletId = PalletId(*b"shldpool"); pub const MaxTreeDepth: u32 = 20; pub const MaxHistoricRoots: u32 = 100; + pub const MaxLeavesPerTree: u32 = 8; pub const MinShieldAmount: u128 = 100; pub const MaxProofSize: u32 = 256; pub const MaxPublicInputs: u32 = 10; @@ -165,6 +166,7 @@ impl pallet_shielded_pool::Config for Test { type PalletId = ShieldedPoolPalletId; type MaxTreeDepth = MaxTreeDepth; type MaxHistoricRoots = MaxHistoricRoots; + type MaxLeavesPerTree = MaxLeavesPerTree; type MinShieldAmount = MinShieldAmount; type WeightInfo = (); type Relayer = pallet_relayer::Pallet; diff --git a/frame/shielded-pool/src/runtime_api_impl.rs b/frame/shielded-pool/src/runtime_api_impl.rs index fb837d1f..70240a1d 100644 --- a/frame/shielded-pool/src/runtime_api_impl.rs +++ b/frame/shielded-pool/src/runtime_api_impl.rs @@ -30,6 +30,32 @@ impl Pallet { crate::merkle::MerkleTreeService::get_merkle_path::(leaf_index) } + /// Forest summary: (active_root, global_size, depth, current_tree_id, + /// sealed_tree_count). + pub fn get_forest_info() -> (Hash, u32, u32, u32, u32) { + let root = crate::storage::MerkleRepository::get_poseidon_root::(); + let size = crate::storage::MerkleRepository::get_tree_size::(); + let cap = T::MaxLeavesPerTree::get(); + (root, size, T::MaxTreeDepth::get(), size / cap, size / cap) + } + + /// Root the leaf's tree anchors to: sealed trees resolve to their + /// permanent final root, the active tree to the live PoseidonRoot. + pub fn get_root_for_leaf(leaf_index: u32) -> Option<(Hash, u32)> { + let size = crate::storage::MerkleRepository::get_tree_size::(); + if leaf_index >= size { + return None; + } + let cap = T::MaxLeavesPerTree::get(); + let tree_id = leaf_index / cap; + let root = if tree_id == size / cap { + crate::storage::MerkleRepository::get_poseidon_root::() + } else { + crate::storage::MerkleRepository::get_sealed_root::(tree_id)? + }; + Some((root, tree_id)) + } + /// Get Merkle proof for a given commitment. /// /// O(1) reverse-index lookup plus an O(depth) sibling-path read from @@ -115,6 +141,52 @@ mod tests { }); } + #[test] + fn get_root_for_leaf_distinguishes_sealed_and_active_trees() { + new_test_ext().execute_with(|| { + // Fill tree 0 (mock cap = 8) and put one leaf in tree 1. + for i in 0..9u8 { + assert_ok!(crate::Pallet::::insert_leaf(Commitment::new( + [i + 1; 32] + ))); + } + let sealed = crate::storage::MerkleRepository::get_sealed_root::(0).unwrap(); + let active = crate::storage::MerkleRepository::get_poseidon_root::(); + + assert_eq!( + crate::Pallet::::get_root_for_leaf(0), + Some((sealed, 0)) + ); + assert_eq!( + crate::Pallet::::get_root_for_leaf(7), + Some((sealed, 0)) + ); + assert_eq!( + crate::Pallet::::get_root_for_leaf(8), + Some((active, 1)) + ); + assert_eq!(crate::Pallet::::get_root_for_leaf(9), None); + }); + } + + #[test] + fn get_forest_info_counts_sealed_trees() { + new_test_ext().execute_with(|| { + for i in 0..9u8 { + assert_ok!(crate::Pallet::::insert_leaf(Commitment::new( + [i + 1; 32] + ))); + } + let (root, size, depth, current_tree_id, sealed_count) = + crate::Pallet::::get_forest_info(); + assert_eq!( + root, + crate::storage::MerkleRepository::get_poseidon_root::() + ); + assert_eq!((size, depth, current_tree_id, sealed_count), (9, 20, 1, 1)); + }); + } + #[test] fn get_merkle_proof_for_commitment_none_when_not_found() { new_test_ext().execute_with(|| { diff --git a/frame/shielded-pool/src/storage.rs b/frame/shielded-pool/src/storage.rs index 6b27b99c..b7251fb9 100644 --- a/frame/shielded-pool/src/storage.rs +++ b/frame/shielded-pool/src/storage.rs @@ -7,8 +7,8 @@ use crate::{ pallet::{ Assets, BalanceOf, CommitmentMemos, CommitmentToLeafIndex, Config, HistoricPoseidonRoots, HistoricRootsOrder, MerkleLeaves, MerkleNodes, MerkleTreeFrontier, MerkleTreeSize, - NextAssetId, NullifierSet, PoolBalancePerAsset, PoseidonRoot, TotalCommitmentsInserted, - TotalNullifiersSpent, + NextAssetId, NullifierSet, PoolBalancePerAsset, PoseidonRoot, SealedRootIndex, + SealedTreeRoots, TotalCommitmentsInserted, TotalNullifiersSpent, }, types::{AssetMetadata, Commitment, EncryptedMemo, Hash}, }; @@ -104,7 +104,14 @@ impl MerkleRepository { HistoricPoseidonRoots::::get(root) } pub fn is_known_root(root: &Hash) -> bool { - Self::is_known_poseidon_root::(root) + Self::is_known_poseidon_root::(root) || SealedRootIndex::::contains_key(root) + } + pub fn insert_sealed_root(tree_id: u32, root: Hash) { + SealedTreeRoots::::insert(tree_id, root); + SealedRootIndex::::insert(root, tree_id); + } + pub fn get_sealed_root(tree_id: u32) -> Option { + SealedTreeRoots::::get(tree_id) } pub fn add_historic_poseidon_root(root: Hash) { HistoricPoseidonRoots::::insert(root, true); diff --git a/template/runtime/src/lib.rs b/template/runtime/src/lib.rs index 306cc7c7..de44bf1e 100644 --- a/template/runtime/src/lib.rs +++ b/template/runtime/src/lib.rs @@ -153,7 +153,10 @@ pub type CheckedExtrinsic = pub type SignedPayload = generic::SignedPayload; /// Storage migrations run on runtime upgrade, oldest first. -pub type Migrations = (pallet_shielded_pool::migrations::v1::MigrateToV1,); +pub type Migrations = ( + pallet_shielded_pool::migrations::v1::MigrateToV1, + pallet_shielded_pool::migrations::v2::MigrateToV2, +); /// Executive: handles dispatch to the various modules. pub type Executive = frame_executive::Executive< @@ -678,6 +681,8 @@ impl pallet_shielded_pool::Config for Runtime { type MaxTreeDepth = ConstU32<20>; /// Historic roots: allows proofs against past states (30s window) type MaxHistoricRoots = ConstU32<100>; + // Pinned to 2^20: clients derive tree_id = leaf_index >> 20 from this. + type MaxLeavesPerTree = ConstU32<1_048_576>; /// Minimum shield amount: prevents spam, 1 ORB = 1e18 wei type MinShieldAmount = ConstU128<1_000_000_000_000_000_000>; type WeightInfo = pallet_shielded_pool::weights::SubstrateWeight; @@ -1319,6 +1324,14 @@ impl_runtime_apis! { ShieldedPool::get_merkle_proof(leaf_index) } + fn get_forest_info() -> ([u8; 32], u32, u32, u32, u32) { + ShieldedPool::get_forest_info() + } + + fn get_root_for_leaf(leaf_index: u32) -> Option<([u8; 32], u32)> { + ShieldedPool::get_root_for_leaf(leaf_index) + } + fn get_merkle_proof_for_commitment( commitment: pallet_shielded_pool::Hash, ) -> Option<(u32, pallet_shielded_pool::DefaultMerklePath)> { diff --git a/ts-tests/test-forest-e2e.ts b/ts-tests/test-forest-e2e.ts new file mode 100644 index 00000000..4a29aabc --- /dev/null +++ b/ts-tests/test-forest-e2e.ts @@ -0,0 +1,106 @@ +/** + * Multi-tree forest E2E against a local dev node built with + * MaxLeavesPerTree = 8 (local-only runtime tweak). + * + * Shields 10 notes to cross the first seal, then asserts: + * - TreeSealed { tree_id: 0 } fired with the expected final root + * - leaves 0..7 anchor to the sealed root (tree_id 0) via privacy RPC + * - leaves 8..9 anchor to the live root (tree_id 1) + * - privacy_getMerkleRoot equals the active-tree root + * + * Run: npx ts-node test-forest-e2e.ts [ws://127.0.0.1:9944] + */ +import { ApiPromise, WsProvider } from "@polkadot/api"; +import { Keyring } from "@polkadot/keyring"; +import { cryptoWaitReady } from "@polkadot/util-crypto"; +import assert from "assert"; + +const WS_URL = process.argv[2] ?? "ws://127.0.0.1:9944"; +const MEMO_SIZE = 180; +const AMOUNT = 10n ** 18n; // MinShieldAmount + +function commitment(i: number): string { + const b = Buffer.alloc(32); + b.writeUInt32LE(i + 1, 0); + b[31] = 0x42; + return "0x" + b.toString("hex"); +} + +function memo(i: number): string { + // EncryptedMemo is a newtype over BoundedVec; pass exact-length hex. + return "0x" + (i + 1).toString(16).padStart(2, "0") + "00".repeat(MEMO_SIZE - 1); +} + +let provider: WsProvider; + +async function rpc(_api: ApiPromise, method: string, params: unknown[]): Promise { + return provider.send(method, params); +} + +async function main() { + await cryptoWaitReady(); + provider = new WsProvider(WS_URL); + const api = await ApiPromise.create({ provider, noInitWarn: true }); + const alice = new Keyring({ type: "sr25519" }).addFromUri("//Alice"); + + let sealedEvent: { treeId: number; finalRoot: string } | null = null; + + let nonce = (await api.rpc.system.accountNextIndex(alice.address)).toNumber(); + for (let i = 0; i < 10; i++) { + await new Promise((resolve, reject) => { + api.tx.shieldedPool + .shield(0, AMOUNT, commitment(i), memo(i)) + .signAndSend(alice, { nonce: nonce++ }, ({ status, events, dispatchError }) => { + if (dispatchError) { + reject(new Error(`shield ${i}: ${dispatchError.toString()}`)); + } else if (status.isInBlock) { + for (const { event } of events) { + if (event.section === "shieldedPool" && event.method === "TreeSealed") { + const d = event.data.toJSON() as any[]; + sealedEvent = { treeId: d[0], finalRoot: d[1] }; + } + } + resolve(); + } + }) + .catch(reject); + }); + console.log(`shield ${i} in block`); + } + + assert(sealedEvent, "TreeSealed event must fire on the 8th insert"); + assert.strictEqual(sealedEvent!.treeId, 0, "first sealed tree is 0"); + console.log(`TreeSealed: tree 0, final_root ${sealedEvent!.finalRoot}`); + + const activeRoot: string = await rpc(api, "privacy_getMerkleRoot", []); + const proof0 = await rpc(api, "privacy_getMerkleProof", [0]); + const proof7 = await rpc(api, "privacy_getMerkleProof", [7]); + const proof8 = await rpc(api, "privacy_getMerkleProof", [8]); + const proof9 = await rpc(api, "privacy_getMerkleProof", [9]); + + // Sealed tree: permanent anchor, not the live root. + for (const p of [proof0, proof7]) { + assert.strictEqual(p.tree_id, 0); + assert.strictEqual(p.root, sealedEvent!.finalRoot, "sealed leaves anchor to the final root"); + assert.notStrictEqual(p.root, activeRoot); + assert.strictEqual(p.path.length, 20); + } + // Active tree. + for (const p of [proof8, proof9]) { + assert.strictEqual(p.tree_id, 1); + assert.strictEqual(p.root, activeRoot, "active leaves anchor to the live root"); + } + // Same-block consistency of the by-commitment variant. + const byCommit = await rpc(api, "privacy_getMerkleProofByCommitment", [commitment(3)]); + assert.strictEqual(byCommit.leaf_index, 3); + assert.strictEqual(byCommit.tree_id, 0); + assert.strictEqual(byCommit.root, sealedEvent!.finalRoot); + + console.log("FOREST E2E OK: seal fired, sealed leaves anchor to permanent root, active tree serves live root"); + await api.disconnect(); +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); From 66733268148252e9d4ef6e358d2c2ff89d8b55a7 Mon Sep 17 00:00:00 2001 From: nol4lej Date: Thu, 30 Jul 2026 17:32:17 -0400 Subject: [PATCH 2/2] chore(runtime): bump spec_version 4 -> 5 for shielded-pool 0.12.0 --- template/runtime/RUNTIME_VERSIONS.md | 1 + template/runtime/src/lib.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/template/runtime/RUNTIME_VERSIONS.md b/template/runtime/RUNTIME_VERSIONS.md index b7d2b113..3a3a86da 100644 --- a/template/runtime/RUNTIME_VERSIONS.md +++ b/template/runtime/RUNTIME_VERSIONS.md @@ -22,6 +22,7 @@ The genesis reset (`69d1b837`) set `spec_version` back to 1 and | spec | tx | Date | Commit | Change | |------|----|------|--------|--------| +| 5 | 1 | 2026-07-30 | — | shielded-pool 0.12.0: multi-tree forest. Full trees seal (`TreeSealed`, permanent `SealedTreeRoots` anchors) and inserts roll over to a fresh tree — the 2^20-note network ceiling is gone. New Config `MaxLeavesPerTree` (2^20), `STORAGE_VERSION` 2 (`MigrateToV2`, version-only), runtime API v2 (`get_forest_info`, `get_root_for_leaf`). No circuit/extrinsic/ABI changes. | | 4 | 1 | 2026-07-30 | `63caafca` | shielded-pool 0.11.0: `MerkleNodes` storage (internal nodes written on every insert) + `MigrateToV1` migration (backfill, `STORAGE_VERSION` 1). O(depth) Merkle proofs. Weights re-benchmarked. The upgrade block runs the one-shot migration (~3s at ~90k leaves). | | 3 | 1 | 2026-07-27 | `d9d40244` | Build with the metadata hash `CheckMetadataHash` requires. | | 2 | 1 | 2026-07-26 | `c285ac3f` | shielded-pool 0.10.1: `recipient_to_field` reduces mod BN254 r — fixes rejection of most Substrate recipients in `unshield` (consensus halt). | diff --git a/template/runtime/src/lib.rs b/template/runtime/src/lib.rs index de44bf1e..c66c718d 100644 --- a/template/runtime/src/lib.rs +++ b/template/runtime/src/lib.rs @@ -204,7 +204,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { spec_name: Cow::Borrowed("orbinum"), impl_name: Cow::Borrowed("orbinum"), authoring_version: 1, - spec_version: 4, + spec_version: 5, impl_version: 1, apis: RUNTIME_API_VERSIONS, transaction_version: 1,