diff --git a/Cargo.lock b/Cargo.lock index d94267ba7..2cef37de2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2029,7 +2029,7 @@ checksum = "af491d569909a7e4dee0ad7db7f5341fef5c614d5b8ec8cf765732aba3cff681" dependencies = [ "serde", "termcolor", - "unicode-width 0.1.14", + "unicode-width 0.2.2", ] [[package]] @@ -2044,7 +2044,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] @@ -3064,7 +3064,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ab67060fc6b8ef687992d439ca0fa36e7ed17e9a0b16b25b601e8757df720de" dependencies = [ "data-encoding", - "syn 1.0.109", + "syn 2.0.117", ] [[package]] @@ -5648,7 +5648,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.5.10", + "socket2 0.6.3", "system-configuration 0.7.0", "tokio", "tower-service", @@ -7791,9 +7791,9 @@ dependencies = [ [[package]] name = "midnight-ledger" -version = "8.1.0" +version = "8.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2182054f3a43ccabac514448fff2487437be293f517a01afc23905df701e8548" +checksum = "479798394847a6686bb9fabda270d16f8525457e9adcc9d2c3486b2422cc4e1c" dependencies = [ "anyhow", "derive-where", @@ -7809,9 +7809,9 @@ dependencies = [ "midnight-ledger-static", "midnight-onchain-runtime 3.1.0", "midnight-serialize", - "midnight-storage 2.0.1", + "midnight-storage 2.0.2", "midnight-transient-crypto 2.2.0", - "midnight-zswap 8.1.0", + "midnight-zswap 8.1.1", "rand 0.8.5", "rayon", "serde", @@ -7852,7 +7852,7 @@ dependencies = [ "midnight-ledger-static", "midnight-onchain-runtime 4.0.0", "midnight-serialize", - "midnight-storage 2.0.1", + "midnight-storage 2.0.2", "midnight-transient-crypto 2.2.0", "midnight-transient-crypto 3.0.0", "midnight-zkir", @@ -7870,7 +7870,7 @@ dependencies = [ [[package]] name = "midnight-node" -version = "2.0.0" +version = "2.1.0" dependencies = [ "async-trait", "authority-selection-inherents", @@ -8042,22 +8042,24 @@ dependencies = [ "midnight-coin-structure 2.0.1", "midnight-coin-structure 3.0.0", "midnight-ledger 7.0.3", - "midnight-ledger 8.1.0", + "midnight-ledger 8.1.1", "midnight-ledger-v9", "midnight-node-ledger-helpers", "midnight-node-res", "midnight-onchain-runtime 2.0.1", "midnight-onchain-runtime 3.1.0", "midnight-onchain-runtime 4.0.0", + "midnight-onchain-state 3.0.0", + "midnight-onchain-state 4.0.0", "midnight-primitives-ledger", "midnight-serialize", "midnight-storage 1.1.1", - "midnight-storage 2.0.1", + "midnight-storage 2.0.2", "midnight-storage-core", "midnight-transient-crypto 2.2.0", "midnight-transient-crypto 3.0.0", "midnight-zswap 7.0.3", - "midnight-zswap 8.1.0", + "midnight-zswap 8.1.1", "midnight-zswap 9.0.0", "moka 0.11.3", "parity-db 0.5.4", @@ -8093,19 +8095,21 @@ dependencies = [ "midnight-coin-structure 2.0.1", "midnight-coin-structure 3.0.0", "midnight-ledger 7.0.3", - "midnight-ledger 8.1.0", + "midnight-ledger 8.1.1", "midnight-ledger-v9", "midnight-onchain-runtime 2.0.1", "midnight-onchain-runtime 3.1.0", "midnight-onchain-runtime 4.0.0", + "midnight-onchain-state 3.0.0", + "midnight-onchain-state 4.0.0", "midnight-serialize", "midnight-storage 1.1.1", - "midnight-storage 2.0.1", + "midnight-storage 2.0.2", "midnight-transient-crypto 2.2.0", "midnight-transient-crypto 3.0.0", "midnight-zkir", "midnight-zswap 7.0.3", - "midnight-zswap 8.1.0", + "midnight-zswap 8.1.1", "midnight-zswap 9.0.0", "rand 0.8.5", "rayon", @@ -8123,7 +8127,7 @@ dependencies = [ [[package]] name = "midnight-node-metadata" -version = "2.0.0" +version = "2.1.0" dependencies = [ "subxt 0.50.0", "walkdir", @@ -8326,7 +8330,7 @@ dependencies = [ "midnight-onchain-state 3.0.0", "midnight-onchain-vm 3.1.0", "midnight-serialize", - "midnight-storage 2.0.1", + "midnight-storage 2.0.2", "midnight-transient-crypto 2.2.0", "rand 0.8.5", "serde", @@ -8351,7 +8355,7 @@ dependencies = [ "midnight-onchain-state 4.0.0", "midnight-onchain-vm 4.0.0", "midnight-serialize", - "midnight-storage 2.0.1", + "midnight-storage 2.0.2", "midnight-transient-crypto 3.0.0", "rand 0.8.5", "serde", @@ -8390,7 +8394,7 @@ dependencies = [ "midnight-base-crypto", "midnight-coin-structure 2.0.1", "midnight-serialize", - "midnight-storage 2.0.1", + "midnight-storage 2.0.2", "midnight-transient-crypto 2.2.0", "rand 0.8.5", "serde", @@ -8408,7 +8412,7 @@ dependencies = [ "midnight-base-crypto", "midnight-coin-structure 3.0.0", "midnight-serialize", - "midnight-storage 2.0.1", + "midnight-storage 2.0.2", "midnight-transient-crypto 2.2.0", "midnight-transient-crypto 3.0.0", "rand 0.8.5", @@ -8452,7 +8456,7 @@ dependencies = [ "midnight-coin-structure 2.0.1", "midnight-onchain-state 3.0.0", "midnight-serialize", - "midnight-storage 2.0.1", + "midnight-storage 2.0.2", "midnight-transient-crypto 2.2.0", "rand 0.8.5", "rpds 1.2.1", @@ -8473,7 +8477,7 @@ dependencies = [ "midnight-coin-structure 3.0.0", "midnight-onchain-state 4.0.0", "midnight-serialize", - "midnight-storage 2.0.1", + "midnight-storage 2.0.2", "midnight-transient-crypto 3.0.0", "rand 0.8.5", "rpds 1.2.1", @@ -8699,12 +8703,13 @@ dependencies = [ [[package]] name = "midnight-storage" -version = "2.0.1" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "210e601a89aee79ce2b007cf96c1a56c23baa306b0c40b9b98d12c27e18d1981" +checksum = "13f3f6a6f45732db644df3eb723a9fc3877490d47f937ea833609ed6cb1508cf" dependencies = [ "crypto", "derive-where", + "hashbrown 0.16.1", "midnight-base-crypto", "midnight-serialize", "midnight-storage-core", @@ -8726,6 +8731,7 @@ dependencies = [ "crypto", "derive-where", "fake", + "hashbrown 0.16.1", "hex", "itertools 0.14.0", "konst 0.4.3", @@ -8929,9 +8935,9 @@ dependencies = [ [[package]] name = "midnight-zswap" -version = "8.1.0" +version = "8.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c83f946d8ac03abea38ca470599801fa08e91e2ddf3cb0963027a7701180c7c" +checksum = "07ff4a06f9d1dcb857be73dd926fcb233b61af3d2a8ab0103b5e39725cd4feb8" dependencies = [ "derive-where", "fake", @@ -8944,7 +8950,7 @@ dependencies = [ "midnight-ledger-static", "midnight-onchain-runtime 3.1.0", "midnight-serialize", - "midnight-storage 2.0.1", + "midnight-storage 2.0.2", "midnight-transient-crypto 2.2.0", "rand 0.8.5", "serde", @@ -8969,7 +8975,7 @@ dependencies = [ "midnight-ledger-static", "midnight-onchain-runtime 4.0.0", "midnight-serialize", - "midnight-storage 2.0.1", + "midnight-storage 2.0.2", "midnight-transient-crypto 2.2.0", "midnight-transient-crypto 3.0.0", "midnight-zkir", @@ -9877,7 +9883,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d8fae84b431384b68627d0f9b3b1245fcf9f46f6c0e3dc902e9dce64edd1967" dependencies = [ "libc", - "windows-sys 0.45.0", + "windows-sys 0.61.2", ] [[package]] @@ -12426,8 +12432,8 @@ version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ - "heck 0.4.1", - "itertools 0.10.5", + "heck 0.5.0", + "itertools 0.14.0", "log", "multimap", "once_cell", @@ -12446,8 +12452,8 @@ version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" dependencies = [ - "heck 0.4.1", - "itertools 0.10.5", + "heck 0.5.0", + "itertools 0.14.0", "log", "multimap", "petgraph 0.8.3", @@ -12479,7 +12485,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.14.0", "proc-macro2", "quote 1.0.45", "syn 2.0.117", @@ -12492,7 +12498,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.14.0", "proc-macro2", "quote 1.0.45", "syn 2.0.117", @@ -12675,7 +12681,7 @@ dependencies = [ "quinn-udp", "rustc-hash 2.1.2", "rustls", - "socket2 0.5.10", + "socket2 0.6.3", "thiserror 2.0.18", "tokio", "tracing", @@ -12713,7 +12719,7 @@ dependencies = [ "cfg_aliases 0.2.1", "libc", "once_cell", - "socket2 0.5.10", + "socket2 0.6.3", "tracing", "windows-sys 0.60.2", ] @@ -20639,7 +20645,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index e6c6c6ede..91634ce44 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -85,11 +85,17 @@ zswap = { version = "=7.0.3", package = "midnight-zswap" } zkir = { version = "^2.2.0", package = "midnight-zkir" } # Ledger 8 (compatible with layout-v2) -# coin-structure and transient-crypto (2.x) share versions with L7 so reuse those entries. -mn-ledger-8 = { version = "=8.1.0", package = "midnight-ledger" } -ledger-storage-ledger-8 = { version = "=2.0.1", package = "midnight-storage", features = ["parity-db"] } +# coin-structure, transient-crypto and zkir share versions with L7 so reuse those entries. +mn-ledger-8 = { version = "=8.1.1", package = "midnight-ledger" } +# `state-translation` (needed by the v8->v9 storage migration) pulls in +# `public-internal-structure`, exposing `merkle_patricia_trie`/`storable`/the +# `state_translation` module used by `midnight_node_ledger::state_translation_v8_to_v9`. +ledger-storage-ledger-8 = { version = "=2.0.2", package = "midnight-storage", features = ["parity-db", "state-translation"] } onchain-runtime-ledger-8 = { version = "=3.1.0", package = "midnight-onchain-runtime" } -zswap-ledger-8 = { version = "=8.1.0", package = "midnight-zswap" } +# Same `midnight-onchain-state` instance that `mn-ledger-8`'s `ContractState` +# resolves to (via onchain-runtime 3.1.0); used as the v8 side of the state translation table. +onchain-state-ledger-8 = { version = "=3.0.0", package = "midnight-onchain-state" } +zswap-ledger-8 = { version = "=8.1.1", package = "midnight-zswap" } # Ledger 9 (compatible with layout-v2; midnight-ledger-v9 crate) # storage shares versions with L8 so reuses that entry. coin-structure and @@ -97,6 +103,9 @@ zswap-ledger-8 = { version = "=8.1.0", package = "midnight-zswap" } # (midnight-zkir 2.2.0) serves all of L7/L8/L9. mn-ledger-9 = { version = "=1.0.0", package = "midnight-ledger-v9" } onchain-runtime-ledger-9 = { version = "=4.0.0", package = "midnight-onchain-runtime" } +# v9 side of the state translation table (matches `mn-ledger-9`'s `ContractState` +# via onchain-runtime 4.0.0). Patched to onchain-state-4.0.0-rc.3 by [patch.crates-io]. +onchain-state-ledger-9 = { version = "=4.0.0", package = "midnight-onchain-state" } zswap-ledger-9 = { version = "=9.0.0", package = "midnight-zswap" } coin-structure-ledger-9 = { version = "=3.0.0", package = "midnight-coin-structure" } transient-crypto-ledger-9 = { version = "=3.0.0", package = "midnight-transient-crypto" } diff --git a/Earthfile b/Earthfile index 77360edd5..9dbb118de 100644 --- a/Earthfile +++ b/Earthfile @@ -1415,7 +1415,7 @@ toolkit-image: tar -xJf node.tar.xz -C /usr/local --strip-components=1 && \ rm node.tar.xz && \ node --version && npm --version && \ - npm install -g npm@11.11.0 && npm --version + npm install -g npm@11.18.0 && npm --version # Add toolkit-js (only when INCLUDE_TOOLKIT_JS=true) IF [ "$INCLUDE_TOOLKIT_JS" = "true" ] @@ -1475,7 +1475,7 @@ audit-npm: curl -fsSL https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-${NODE_ARCH}.tar.xz -o node.tar.xz && \ tar -xJf node.tar.xz -C /usr/local --strip-components=1 && \ rm node.tar.xz && \ - npm install -g npm@11.11.0 && \ + npm install -g npm@11.18.0 && \ node --version && npm --version COPY ${DIRECTORY} ${DIRECTORY} @@ -1514,7 +1514,7 @@ audit-yarn: curl -fsSL https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-${NODE_ARCH}.tar.xz -o node.tar.xz && \ tar -xJf node.tar.xz -C /usr/local --strip-components=1 && \ rm node.tar.xz && \ - npm install -g npm@11.11.0 && \ + npm install -g npm@11.18.0 && \ node --version && npm --version # Install and enable corepack for yarn support @@ -1565,7 +1565,7 @@ fix-lock-npm: curl -fsSL https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-${NODE_ARCH}.tar.xz -o node.tar.xz && \ tar -xJf node.tar.xz -C /usr/local --strip-components=1 && \ rm node.tar.xz && \ - npm install -g npm@11.11.0 && \ + npm install -g npm@11.18.0 && \ node --version && npm --version COPY ${DIRECTORY}/package.json ${DIRECTORY}/package-lock.json ${DIRECTORY}/ diff --git a/changes/node/changed/hardfork-skew-block-ledger-reads.md b/changes/node/changed/hardfork-skew-block-ledger-reads.md new file mode 100644 index 000000000..d03b2d27b --- /dev/null +++ b/changes/node/changed/hardfork-skew-block-ledger-reads.md @@ -0,0 +1,36 @@ +#node #ledger +# Serve ledger state reads at the ledger-hardfork `set_code` block + +On a chain that hardforks ledger 8 -> 9 via a governance `set_code`, exactly one historical +block was permanently unreadable: every ledger state read at that hash failed with +`Deserialization(TypedArenaKey)`. The pre-fork runtime shipped `system_version: 1`, so +`frame_system` overwrote `:code` *inside* the `set_code` block while pallet-midnight's v8 -> v9 +state translation only ran in the next block's `initialize_block`. That block's committed state +therefore pairs ledger-9 `:code` with a ledger-8 `StateKey` forever, and any read at that hash +executes ledger-9 code against a ledger-8 arena root. + +The read-only accessors of the ledger-9 host API now check the tagged-serialization header of +the `state_key` they are handed. If it is a ledger-8 arena root, the read is served from the +ledger-8 bridge instead — v8 and v9 share one storage crate and hence one arena, so this is a +pure dispatch with no data movement. The `StateKey` tag is the signal rather than the pallet +storage version, because the 2.0.0 runtime already ran ledger 9 while still reporting +pallet-midnight storage version 1. + +Because the dispatch sits in the host function, it covers every caller of the affected reads — +the `midnight_*` JSON-RPCs, `MidnightRuntimeApi` through `state_call`, and subxt-based tooling +such as `chain-indexer` — rather than only the node's own RPC layer. Affected reads: +`get_contract_state`, `get_zswap_chain_state`, `get_zswap_state_root`, `get_ledger_state_root`, +`get_ledger_parameters`, `get_c_to_m_bridge_min_amount`, `get_unclaimed_amount`, +`get_bridge_receiving_amount`. + +Not covered, deliberately: the transaction paths (`get_transaction_cost`, +`validate_transaction`, `apply_transaction`). At the skew block those concern ledger-9-format +transactions, which ledger-8 code cannot deserialize in any case, and they resolve on their own +one block later once the migration has run. + +Behaviour note: at the `set_code` block `midnight_ledgerStateRoot` now returns a **v8-tagged** +root — correct, since that block's state *is* v8 — so consumers walking the fork see the tag +flip at the migration block rather than a hole. + +PR: https://github.com/midnightntwrk/midnight-node/pull/1985 +Issue: https://github.com/midnightntwrk/midnight-node/issues/1959 diff --git a/changes/runtime/changed/ledger-8-to-9-hardfork-migration.md b/changes/runtime/changed/ledger-8-to-9-hardfork-migration.md new file mode 100644 index 000000000..e2c92f72c --- /dev/null +++ b/changes/runtime/changed/ledger-8-to-9-hardfork-migration.md @@ -0,0 +1,22 @@ +#node #runtime #ledger + +# On-chain ledger 8->9 hardfork state migration + +Lets a ledger-8 chain (e.g. `1.0.1`) runtime-upgrade in place to the current +ledger-9 runtime. A new host fn, `migrate_state_v8_to_v9`, runs the +`StateTranslationTable` (ported from `midnight-ledger` PR #539) to translate +the on-chain `LedgerState` from v13 to v18. It's wired in as +`pallet_midnight::migrations::v2::MigrateV1ToV2`, a `VersionedMigration<1,2,..>` +that fires once when a ledger-8 chain (pallet-midnight storage version 1) +upgrades to this runtime (storage version 2); a fresh ledger-9 genesis starts +at version 2 and skips it. The migration's weight is derived from the ledger +cost model rather than a hand-tuned estimate. + +Also includes two fixes needed to support both ledger-8 and ledger-9 chains: +version-aware genesis seeding (detected via `serialize::peek_tag` instead of +hardcoding the v9 deserializer), and restoring the ledger-8 +`construct_distribute_treasury_system_tx` host fn, which the `1.0.1` WASM +still imports. + +PR: https://github.com/midnightntwrk/midnight-node/pull/1925 +Issue: https://github.com/midnightntwrk/midnight-node/issues/1580 diff --git a/changes/toolkit/changed/bump-npm-sbom-tar-critical.md b/changes/toolkit/changed/bump-npm-sbom-tar-critical.md new file mode 100644 index 000000000..0af6f436e --- /dev/null +++ b/changes/toolkit/changed/bump-npm-sbom-tar-critical.md @@ -0,0 +1,17 @@ +#toolkit #security +# Bump bundled npm 11.11.0 -> 11.18.0 to clear toolkit image SBOM findings + +The Grype scan of the toolkit image failed on a critical `tar` advisory +(GHSA-23hp-3jrh-7fpw, `tar@7.5.9`, fixed in 7.5.19). That `tar`, along with the +other flagged npm packages (sigstore, @sigstore/core, @sigstore/verify, +minimatch, brace-expansion, picomatch, ip-address), is vendored inside the +globally-installed npm CLI, not in toolkit-js's dependencies. + +- Bumped the pinned `npm install -g npm@11.11.0` to `npm@11.18.0` across the + Earthfile targets (`toolkit-image`, `audit-npm`, `audit-yarn`, + `fix-lock-npm`). npm 11.18.0 bundles `tar@7.5.19` (clearing the critical) plus + patched versions of every other flagged npm package. +- Minor bump within the 11.x line; engine requirement is unchanged + (`^20.17.0 || >=22.9.0`), satisfied by the image's Node 24.18.0. + +PR: https://github.com/midnightntwrk/midnight-node/pull/1919 diff --git a/changes/toolkit/changed/hardfork-fork-aware-tx-generation.md b/changes/toolkit/changed/hardfork-fork-aware-tx-generation.md new file mode 100644 index 000000000..9fc711c0e --- /dev/null +++ b/changes/toolkit/changed/hardfork-fork-aware-tx-generation.md @@ -0,0 +1,18 @@ +#toolkit #ledger9 + +# Fork-aware transaction generation across the ledger 8->9 hardfork + +`replay_blocks` now detects the v8->v9 fork boundary and runs the same +`StateTranslationTable` translation used by the on-chain migration +(`fork_context_8_to_9` / `fork_8_to_9_if_needed`), so transactions generated +after the fork are built against the correctly-translated ledger-9 context +instead of a stale ledger-8 one. + +The `runtime-upgrade` command now waits for the new runtime to actually +*execute* at a finalized block, not just be applied/stored. The stored spec +version flips at the apply block, but that block still executes under the +old runtime, so polling only the stored spec left transaction generation +reading a block short of any ledger-9-classified block. + +PR: https://github.com/midnightntwrk/midnight-node/pull/1925 +Issue: https://github.com/midnightntwrk/midnight-node/issues/1580 diff --git a/docs/openrpc.json b/docs/openrpc.json index 9b4e15f90..d758dabdb 100644 --- a/docs/openrpc.json +++ b/docs/openrpc.json @@ -2,7 +2,7 @@ "openrpc": "1.4.0", "info": { "title": "Midnight Node JSON-RPC API", - "version": "2.0.0", + "version": "2.1.0", "description": "JSON-RPC API for the Midnight privacy blockchain node. Custom methods provide access to the privacy ledger, governance parameters, and peer management. Standard Substrate methods are also listed." }, "methods": [ diff --git a/ledger/Cargo.toml b/ledger/Cargo.toml index 4a09698cd..023ea32a9 100644 --- a/ledger/Cargo.toml +++ b/ledger/Cargo.toml @@ -18,11 +18,13 @@ zswap = { workspace = true, optional = true } mn-ledger-8 = { workspace = true, features = ["proving"], optional = true } onchain-runtime-ledger-8 = { workspace = true, optional = true } +onchain-state-ledger-8 = { workspace = true, optional = true } ledger-storage-ledger-8 = { workspace = true, optional = true } zswap-ledger-8 = { workspace = true, optional = true } mn-ledger-9 = { workspace = true, features = ["proving"], optional = true } onchain-runtime-ledger-9 = { workspace = true, optional = true } +onchain-state-ledger-9 = { workspace = true, optional = true } zswap-ledger-9 = { workspace = true, optional = true } coin-structure-ledger-9 = { workspace = true, optional = true } transient-crypto-ledger-9 = { workspace = true, optional = true } @@ -50,6 +52,11 @@ scale-info.workspace = true [dev-dependencies] midnight-node-res = { workspace = true, features = ["test", "chain-spec"] } +# The crate's own `#[cfg(test)]` modules (api::ledger, api::transaction, and the +# state-translation tests) use `extract_tx_with_context`, gated behind the +# helpers `can-panic` feature. Enable it for test builds so the tests compile +# when the crate is tested standalone. +midnight-node-ledger-helpers = { workspace = true, features = ["can-panic", "test-utils"] } [features] default = [ @@ -82,10 +89,12 @@ std = [ "zswap", "mn-ledger-8", "onchain-runtime-ledger-8", + "onchain-state-ledger-8", "ledger-storage-ledger-8", "zswap-ledger-8", "mn-ledger-9", "onchain-runtime-ledger-9", + "onchain-state-ledger-9", "zswap-ledger-9", "coin-structure-ledger-9", "transient-crypto-ledger-9", diff --git a/ledger/helpers/Cargo.toml b/ledger/helpers/Cargo.toml index 0c675d892..e46fde9c5 100644 --- a/ledger/helpers/Cargo.toml +++ b/ledger/helpers/Cargo.toml @@ -21,11 +21,13 @@ reqwest = { workspace = true } mn-ledger-8 = { workspace = true, features = ["proving"] } onchain-runtime-ledger-8 = { workspace = true } +onchain-state-ledger-8 = { workspace = true } ledger-storage-ledger-8 = { workspace = true } zswap-ledger-8 = { workspace = true } mn-ledger-9 = { workspace = true, features = ["proving", "test-utilities"] } onchain-runtime-ledger-9 = { workspace = true } +onchain-state-ledger-9 = { workspace = true } zswap-ledger-9 = { workspace = true } coin-structure-ledger-9 = { workspace = true } transient-crypto-ledger-9 = { workspace = true } diff --git a/ledger/helpers/src/fork/fork_8_to_9.rs b/ledger/helpers/src/fork/fork_8_to_9.rs index 918cda2e9..fac03ec49 100644 --- a/ledger/helpers/src/fork/fork_8_to_9.rs +++ b/ledger/helpers/src/fork/fork_8_to_9.rs @@ -2,6 +2,10 @@ use std::collections::HashMap; use tokio::sync::Mutex as MutexTokio; +use crate::state_translation_v8_to_v9::StateTranslationTable; +use base_crypto::cost_model::CostDuration; +use ledger_storage_ledger_8::state_translation::TypedTranslationState; + type Db8 = crate::ledger_8::DefaultDB; type Db9 = crate::ledger_9::DefaultDB; @@ -67,8 +71,35 @@ pub fn fork_context_8_to_9( context8: LedgerContext8, ) -> Result, std::io::Error> { let ledger_state_8 = context8.ledger_state.lock().expect("failed to lock ledger state"); - let ledger_state: crate::ledger_9::Sp, Db8> = - old_to_new_sp(ledger_state_8.clone())?; + // Real v8->v9 state translation (NOT `old_to_new_sp`): the LedgerState tag + // changed v13->v18 and its shape changed, so a bare arena-key reuse would + // produce a v9 root the ledger-9 machinery can't read. Walk the v8 state + // through the same `StateTranslationTable` the on-chain migration uses. Db8 + // == Db9 (both `ledger_storage_ledger_8::DefaultDB`), so source and target + // share one arena and a single-`D` `TypedTranslationState` applies. + let ledger_state: crate::ledger_9::Sp, Db8> = { + let mut tl = TypedTranslationState::< + mn_ledger_8::structure::LedgerState, + mn_ledger_9::structure::LedgerState, + StateTranslationTable, + Db8, + >::start(ledger_state_8.clone())?; + // Single-shot: a generous per-step budget drains the whole state in a + // couple of iterations; the step cap is only a runaway backstop. + // 1_000_000_000_000 pico-seconds == 1 second + let budget = CostDuration::from_picoseconds(1_000_000_000_000); + let mut steps = 0usize; + loop { + steps += 1; + if steps > 100_000 { + return Err(std::io::Error::other("v8->v9 state translation did not converge")); + } + tl = tl.run(budget)?; + if let Some(result) = tl.result()? { + break result; + } + } + }; let mut wallets = HashMap::new(); for (k, v) in context8.wallets.lock().expect("failed to lock wallets").iter() { diff --git a/ledger/helpers/src/lib.rs b/ledger/helpers/src/lib.rs index 06494e10f..e2f67bdf2 100644 --- a/ledger/helpers/src/lib.rs +++ b/ledger/helpers/src/lib.rs @@ -16,6 +16,11 @@ mod utils; pub use utils::find_dependency_version; pub mod extract_tx_with_context; +/// v8 -> v9 ledger state translation table (ported from midnight-ledger PR #539). +/// Consumed by the runtime storage migration (via the `ledger` crate) and by the +/// toolkit fork boundary (`fork::fork_8_to_9`). +pub mod state_translation_v8_to_v9; + /// Strategy for ordering candidate coins/UTXOs during input selection. /// /// Defined at the crate root (not inside the version-specific `common` module) so that diff --git a/ledger/helpers/src/state_translation_v8_to_v9.rs b/ledger/helpers/src/state_translation_v8_to_v9.rs new file mode 100644 index 000000000..4106b7819 --- /dev/null +++ b/ledger/helpers/src/state_translation_v8_to_v9.rs @@ -0,0 +1,710 @@ +// This file is part of midnight-node. +// Copyright (C) 2025-2026 Midnight Foundation +// SPDX-License-Identifier: Apache-2.0 +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! State translation from ledger v8 to ledger v9. +//! +//! Ported from `midnight-ledger` PR #539 (`v8-to-v9-state-translation`). The +//! only changes from the upstream crate are the import aliases below, which map +//! the translation's `ledger_v8` / `ledger_v9` / `onchain_state_v8` / +//! `onchain_state_v9` / `storage` / `serialize` crate names onto this +//! workspace's package aliases. The [`StateTranslationTable`] is consumed by the +//! v8->v9 storage migration ([`crate::host_api::migration_8_to_9`]). +//! +//! ## State shape differences (only stored types listed) +//! +//! | type | v8 tag | v9 tag | change | +//! | ---------------------------- | ------------------------------------ | ------------------------------------ | ------ | +//! | LedgerState | `ledger-state[v13]` | `ledger-state[v18]` | `bridge_receiving` map gains `NightAnn` | +//! | LedgerParameters | `ledger-parameters[v5]` | `ledger-parameters[v8]` | adds `min_block_price`; `TransactionLimits` adds `max_contract_metadata_size`; `TransactionCostModel` drops `parallelism_factor`, adds `validation`/`guaranteed`/`fallible` factors | +//! | ContractState | `contract-state[v6]` | `contract-state[v8]` | reflows `ContractOperation` + `ContractMaintenanceAuthority` changes | +//! | ContractOperation | `contract-operation[v4]` | `contract-operation[v6]` | single `v2` key -> `{ v2, v3, ir }`; v8 key maps to `v2`, new `v3`/`ir` empty | +//! | ContractMaintenanceAuthority | `contract-maintenance-authority[v1]` | `contract-maintenance-authority[v2]` | `committee: Vec` -> `Vec` (Schnorr/ECDSA sum) | +//! +//! Everything else (zswap, utxo, dust, replay_protection, treasury, +//! unclaimed_block_rewards) is tag-stable and passes through `recast`. + +// Map the upstream translation crate names onto the node workspace's package +// aliases. `mn-ledger-8`/`mn-ledger-9` are the two `midnight-ledger` majors; +// `onchain-state-ledger-8`/`-9` are the exact `midnight-onchain-state` instances +// that each ledger's `ContractState` resolves to; `ledger-storage-ledger-8` +// (midnight-storage 2.0.1, `state-translation` feature) backs both. +use ledger_storage_ledger_8 as storage; +use midnight_serialize as serialize; +use mn_ledger_8 as ledger_v8; +use mn_ledger_9 as ledger_v9; +use onchain_state_ledger_8 as onchain_state_v8; +use onchain_state_ledger_9 as onchain_state_v9; + +use base_crypto::cost_model::CostDuration; +use serialize::Tagged; +use std::ops::Deref; +use std::{any::Any, borrow::Cow, io, marker::PhantomData}; +use storage::{ + Storable, + arena::Sp, + db::DB, + merkle_patricia_trie::{self, Annotation, MerklePatriciaTrie}, + state_translation::*, + storable::SizeAnn, + storage::{HashMap, Map, default_storage}, +}; + +// ---------- Generic helpers (copied from the v6->v7 reference) ---------- + +/// Recast a stored object from one type to another, requiring matching tags. +/// Used for subtrees whose tag is unchanged between v8 and v9. +fn recast + Tagged, B: Storable + Tagged, D: DB>( + a: &Sp, +) -> io::Result> { + if A::tag() != B::tag() { + return io::Result::Err(io::Error::other("tags do not match")); + } + default_storage::().get_lazy(&a.as_child().into()) +} + +/// Generic MPT translation: walks the trie, translating each entry via the +/// table-registered translation for `A->B`, and recomputes annotations under +/// `AnnB` from the new values. +struct MptTl(PhantomData<(A, B, AnnA, AnnB)>); + +impl< + A: Storable + Tagged, + B: Storable + Tagged, + AnnA: Annotation + Storable + Tagged, + AnnB: Annotation + Storable + Tagged, + D: DB, +> DirectTranslation, MerklePatriciaTrie, D> + for MptTl +{ + fn required_translations() -> Vec { + vec![TranslationId( + merkle_patricia_trie::Node::::tag(), + merkle_patricia_trie::Node::::tag(), + )] + } + fn child_translations( + source: &MerklePatriciaTrie, + ) -> Vec<(TranslationId, Sp)> { + let tlids = , _, D>>::required_translations(); + vec![(tlids[0].clone(), source.0.upcast())] + } + fn finalize( + source: &MerklePatriciaTrie, + _limit: &mut CostDuration, + cache: &TranslationCache, + ) -> io::Result>> { + let tls = Self::child_translations(source); + Ok(Some(MerklePatriciaTrie(try_resopt!(cache.resolve(&tls[0].0, tls[0].1.as_child()))))) + } +} + +impl< + A: Storable + Tagged, + B: Storable + Tagged, + AnnA: Storable + Tagged + Annotation, + AnnB: Storable + Tagged + Annotation, + D: DB, +> + DirectTranslation< + merkle_patricia_trie::Node, + merkle_patricia_trie::Node, + D, + > for MptTl +{ + fn required_translations() -> Vec { + let entry_tl = TranslationId(A::tag(), B::tag()); + let self_tl = TranslationId( + merkle_patricia_trie::Node::::tag(), + merkle_patricia_trie::Node::::tag(), + ); + vec![entry_tl, self_tl] + } + fn child_translations( + source: &merkle_patricia_trie::Node, + ) -> Vec<(TranslationId, Sp)> { + let tls = , _, D>>::required_translations(); + let entry_tl = tls[0].clone(); + let self_tl = tls[1].clone(); + match source { + merkle_patricia_trie::Node::Empty => vec![], + merkle_patricia_trie::Node::Branch { children, .. } => { + children.iter().map(|child| (self_tl.clone(), child.upcast())).collect() + }, + merkle_patricia_trie::Node::Extension { child, .. } => { + vec![(self_tl, child.upcast())] + }, + merkle_patricia_trie::Node::MidBranchLeaf { value, child, .. } => { + vec![(entry_tl, value.upcast()), (self_tl, child.upcast())] + }, + merkle_patricia_trie::Node::Leaf { value, .. } => vec![(entry_tl, value.upcast())], + } + } + fn finalize( + source: &merkle_patricia_trie::Node, + _limit: &mut CostDuration, + cache: &TranslationCache, + ) -> io::Result>> { + let tls = Self::child_translations(source); + Ok(Some(match source { + merkle_patricia_trie::Node::Empty => merkle_patricia_trie::Node::Empty, + merkle_patricia_trie::Node::Branch { .. } => { + let mut new_children = + core::array::from_fn(|_| Sp::new(merkle_patricia_trie::Node::Empty)); + for (child, new_child) in tls.iter().zip(new_children.iter_mut()) { + *new_child = try_resopt!(cache.resolve(&child.0, child.1.as_child())); + } + let ann = new_children.iter().fold(AnnB::empty(), |acc, x| { + acc.append(&merkle_patricia_trie::Node::::ann(x)) + }); + merkle_patricia_trie::Node::Branch { ann, children: Box::new(new_children) } + }, + merkle_patricia_trie::Node::Extension { compressed_path, .. } => { + let child: Sp, D> = + try_resopt!(cache.resolve(&tls[0].0, tls[0].1.as_child())); + let ann = merkle_patricia_trie::Node::::ann(&child); + merkle_patricia_trie::Node::Extension { + ann, + compressed_path: compressed_path.clone(), + child, + } + }, + merkle_patricia_trie::Node::Leaf { .. } => { + let value = try_resopt!(cache.resolve(&tls[0].0, tls[0].1.as_child())); + let ann = AnnB::from_value(&value); + merkle_patricia_trie::Node::Leaf { ann, value } + }, + merkle_patricia_trie::Node::MidBranchLeaf { .. } => { + let value = try_resopt!(cache.resolve(&tls[0].0, tls[0].1.as_child())); + let child: Sp, D> = + try_resopt!(cache.resolve(&tls[1].0, tls[1].1.as_child())); + let ann = AnnB::from_value(&value) + .append(&merkle_patricia_trie::Node::::ann(&child)); + merkle_patricia_trie::Node::MidBranchLeaf { ann, value, child } + }, + })) + } +} + +/// Identity translation for a type whose serialization is unchanged across +/// versions. Needed when an MPT's entries are tag-stable but its annotation +/// changes (e.g. `bridge_receiving`). +struct IdentityTl(PhantomData); + +impl + Clone, D: DB> DirectTranslation for IdentityTl { + fn required_translations() -> Vec { + Vec::new() + } + fn child_translations(_: &T) -> Vec<(TranslationId, Sp)> { + Vec::new() + } + fn finalize( + source: &T, + _limit: &mut CostDuration, + _cache: &TranslationCache, + ) -> io::Result> { + Ok(Some(source.clone())) + } +} + +// ---------- Translation IDs (shorthand) ---------- + +struct Ids; + +impl Ids { + fn contract_mpt() -> TranslationId { + TranslationId( + MerklePatriciaTrie::< + onchain_state_v8::state::ContractState, + D, + ledger_v8::annotation::NightAnn, + >::tag(), + MerklePatriciaTrie::< + onchain_state_v9::state::ContractState, + D, + ledger_v9::annotation::NightAnn, + >::tag(), + ) + } + + fn bridge_receiving_mpt() -> TranslationId { + TranslationId( + MerklePatriciaTrie::::tag(), + MerklePatriciaTrie::::tag(), + ) + } + + fn parameters() -> TranslationId { + TranslationId( + ledger_v8::structure::LedgerParameters::tag(), + ledger_v9::structure::LedgerParameters::tag(), + ) + } +} + +// ---------- Top-level: LedgerState v8 -> v9 ---------- + +struct LedgerStateTl; + +impl + DirectTranslation, ledger_v9::structure::LedgerState, D> + for LedgerStateTl +{ + fn required_translations() -> Vec { + vec![Ids::parameters(), Ids::bridge_receiving_mpt::(), Ids::contract_mpt::()] + } + + fn child_translations( + source: &ledger_v8::structure::LedgerState, + ) -> Vec<(TranslationId, Sp)> { + vec![ + (Ids::parameters(), source.parameters.upcast()), + (Ids::bridge_receiving_mpt::(), source.bridge_receiving.mpt.upcast()), + (Ids::contract_mpt::(), source.contract.mpt.upcast()), + ] + } + + fn finalize( + source: &ledger_v8::structure::LedgerState, + _limit: &mut CostDuration, + cache: &TranslationCache, + ) -> io::Result>> { + let Some(parameters) = cache.lookup(&Ids::parameters(), source.parameters.as_child()) + else { + return Ok(None); + }; + let Some(bridge_recv_mpt) = + cache.lookup(&Ids::bridge_receiving_mpt::(), source.bridge_receiving.mpt.as_child()) + else { + return Ok(None); + }; + let Some(contract_mpt) = + cache.lookup(&Ids::contract_mpt::(), source.contract.mpt.as_child()) + else { + return Ok(None); + }; + + Ok(Some(ledger_v9::structure::LedgerState { + network_id: source.network_id.clone(), + parameters: parameters.force_downcast(), + locked_pool: source.locked_pool, + bridge_receiving: Map { mpt: bridge_recv_mpt.force_downcast(), key_type: PhantomData }, + reserve_pool: source.reserve_pool, + block_reward_pool: source.block_reward_pool, + unclaimed_block_rewards: Map { + mpt: recast(&source.unclaimed_block_rewards.mpt)?, + key_type: PhantomData, + }, + treasury: Map { mpt: recast(&source.treasury.mpt)?, key_type: PhantomData }, + zswap: recast(&source.zswap)?, + contract: Map { mpt: contract_mpt.force_downcast(), key_type: PhantomData }, + utxo: recast(&source.utxo)?, + replay_protection: recast(&source.replay_protection)?, + dust: recast(&source.dust)?, + })) + } +} + +// ---------- LedgerParameters v8 -> v9 ---------- + +struct LedgerParametersTl; + +impl + DirectTranslation< + ledger_v8::structure::LedgerParameters, + ledger_v9::structure::LedgerParameters, + D, + > for LedgerParametersTl +{ + fn required_translations() -> Vec { + Vec::new() + } + fn child_translations( + _: &ledger_v8::structure::LedgerParameters, + ) -> Vec<(TranslationId, Sp)> { + Vec::new() + } + fn finalize( + source: &ledger_v8::structure::LedgerParameters, + _limit: &mut CostDuration, + _cache: &TranslationCache, + ) -> io::Result> { + // Base-crypto-backed fields (Duration, FixedPoint, primitives) are + // assignable directly because `midnight-base-crypto` is unified across + // v8 and v9 by workspace patches. Composite types defined in `ledger` + // (TransactionCostModel, dust parameters, etc.) are tag-stable but not + // identical types, so we go through the (de)serializer. + // + // `TransactionLimits` is the exception: v9 bumped it to + // `transaction-limits[v3]` by adding `max_contract_metadata_size`, so + // it is no longer tag-stable and is rebuilt field-by-field (its other + // fields are unified base-crypto types). + Ok(Some(ledger_v9::structure::LedgerParameters { + // `TransactionCostModel` bumped `transaction-cost-model[v4]`->`[v5]`: + // v9 drops `parallelism_factor` and adds three `FixedPoint` factors. + // The two surviving fields are tag-stable and recast through; the new + // factors get the v9 INITIAL_PARAMETERS defaults. + cost_model: ledger_v9::structure::TransactionCostModel { + runtime_cost_model: recast_base(&source.cost_model.runtime_cost_model)?, + baseline_cost: recast_base(&source.cost_model.baseline_cost)?, + // NEW IN v9 — placeholder; the production value should match the + // value chosen for the hardfork. + validation_factor: ledger_v9::structure::INITIAL_PARAMETERS + .cost_model + .validation_factor, + guaranteed_factor: ledger_v9::structure::INITIAL_PARAMETERS + .cost_model + .guaranteed_factor, + fallible_factor: ledger_v9::structure::INITIAL_PARAMETERS + .cost_model + .fallible_factor, + }, + limits: ledger_v9::structure::TransactionLimits { + transaction_byte_limit: source.limits.transaction_byte_limit, + time_to_dismiss_per_byte: source.limits.time_to_dismiss_per_byte, + min_time_to_dismiss: source.limits.min_time_to_dismiss, + block_limits: source.limits.block_limits, + block_withdrawal_minimum_multiple: source.limits.block_withdrawal_minimum_multiple, + // NEW IN v9 — placeholder; the production value should match + // the value chosen for the hardfork. + max_contract_metadata_size: ledger_v9::structure::INITIAL_PARAMETERS + .limits + .max_contract_metadata_size, + }, + dust: recast_base(&source.dust)?, + fee_prices: recast_base(&source.fee_prices)?, + global_ttl: source.global_ttl, + cost_dimension_min_ratio: source.cost_dimension_min_ratio, + price_adjustment_a_parameter: source.price_adjustment_a_parameter, + cardano_to_midnight_bridge_fee_basis_points: source + .cardano_to_midnight_bridge_fee_basis_points, + c_to_m_bridge_min_amount: source.c_to_m_bridge_min_amount, + // NEW IN v9 — placeholder; the production value should match the + // value chosen for the hardfork. + min_block_price: ledger_v9::structure::INITIAL_PARAMETERS.min_block_price, + })) + } +} + +/// Recast for tag-stable base types passed by value (cost model, limits, etc.). +/// Not the same as `recast` above which only works for `Sp`. +fn recast_base( + a: &A, +) -> io::Result { + if A::tag() != B::tag() { + return Err(io::Error::other("tags do not match")); + } + let mut buf = Vec::new(); + a.serialize(&mut buf)?; + B::deserialize(&mut &buf[..], 0) +} + +// ---------- ContractOperation v8 -> v9 ---------- + +/// Translate a single contract operation. v9 grew `ContractOperation` from a +/// single `v2` verifier key (`contract-operation[v4]`) to `{ v2, v3, ir }` +/// (`contract-operation[v6]`). v8's only key is a zk-stdlib-v1 key +/// (`verifier-key[v6]`), which v9 keeps in its `v2` slot: that slot is backed by +/// the same `transient-crypto` 2.x crate (`transient_crypto_old`), so it is the +/// identical type and assigns directly. The new zk-stdlib-v2 `v3` key +/// (`verifier-key[v7]`, transient-crypto 3.x) and the `ir` slot have no v8 +/// equivalent and stay empty — v9 keys are *not* synthesized from v8 keys. +/// (Note `ContractOperation::new(vk, ir)` sets `v3`, not `v2`, so the struct is +/// built field-wise here.) +fn translate_contract_operation( + source: &onchain_state_v8::state::ContractOperation, +) -> onchain_state_v9::state::ContractOperation { + // `ContractOperation` is `#[non_exhaustive]`; `new` seeds `v3`/`ir`, and the + // v8 key goes into the `v2` slot field-wise. + let mut op = onchain_state_v9::state::ContractOperation::new(None, None); + op.v2 = source.v2.clone(); + op +} + +// ---------- ContractState v8 -> v9 ---------- + +struct ContractStateTl; + +impl + DirectTranslation< + onchain_state_v8::state::ContractState, + onchain_state_v9::state::ContractState, + D, + > for ContractStateTl +{ + fn required_translations() -> Vec { + Vec::new() + } + fn child_translations( + _: &onchain_state_v8::state::ContractState, + ) -> Vec<(TranslationId, Sp)> { + Vec::new() + } + fn finalize( + source: &onchain_state_v8::state::ContractState, + _limit: &mut CostDuration, + _cache: &TranslationCache, + ) -> io::Result>> { + // `operations` entries (ContractOperation) changed shape, so the map + // is rebuilt entry-by-entry. The translation machinery can't walk these + // base-storable leaves nested under a contract, but a contract's + // operation set is small, so an in-place rebuild is fine. ChargedState + // and the balance map (keyed u128) are tag-stable and recast through. + let mut operations = HashMap::new(); + for entry in source.operations.iter() { + let (key, op) = &*entry; + let key_v9: onchain_state_v9::state::EntryPointBuf = key[..].into(); + operations = operations.insert(key_v9, translate_contract_operation(op)); + } + let committee_v9 = source + .maintenance_authority + .committee + .iter() + .map(|vk| onchain_state_v9::state::ContractMaintenanceVerifyingKey::Schnorr(vk.clone())) + .collect(); + let maintenance_authority = onchain_state_v9::state::ContractMaintenanceAuthority { + committee: committee_v9, + threshold: source.maintenance_authority.threshold, + counter: source.maintenance_authority.counter, + }; + Ok(Some(onchain_state_v9::state::ContractState:: { + data: recast::< + onchain_state_v8::state::ChargedState, + onchain_state_v9::state::ChargedState, + D, + >(&Sp::new(source.data.clone()))? + .deref() + .clone(), + operations, + maintenance_authority, + balance: HashMap(Map { mpt: recast(&source.balance.0.mpt)?, key_type: PhantomData }), + })) + } +} + +// ---------- Translation table ---------- + +pub struct StateTranslationTable; + +impl TranslationTable for StateTranslationTable { + const TABLE: &[(TranslationId, &dyn TypelessTranslation)] = &[ + // Top-level + ( + TranslationId(Cow::Borrowed("ledger-state[v13]"), Cow::Borrowed("ledger-state[v18]")), + &DirectSpTranslation::<_, _, LedgerStateTl, _>(PhantomData), + ), + // LedgerParameters + ( + TranslationId( + Cow::Borrowed("ledger-parameters[v5]"), + Cow::Borrowed("ledger-parameters[v8]"), + ), + &DirectSpTranslation::<_, _, LedgerParametersTl, _>(PhantomData), + ), + // ContractState + ( + TranslationId(Cow::Borrowed("contract-state[v6]"), Cow::Borrowed("contract-state[v8]")), + &DirectSpTranslation::<_, _, ContractStateTl, _>(PhantomData), + ), + // `contract` MPT in LedgerState — entries are ContractState + ( + TranslationId( + Cow::Borrowed("mpt(contract-state[v6],night-annotation)"), + Cow::Borrowed("mpt(contract-state[v8],night-annotation)"), + ), + &DirectSpTranslation::< + MerklePatriciaTrie< + onchain_state_v8::state::ContractState, + D, + ledger_v8::annotation::NightAnn, + >, + MerklePatriciaTrie< + onchain_state_v9::state::ContractState, + D, + ledger_v9::annotation::NightAnn, + >, + MptTl< + onchain_state_v8::state::ContractState, + onchain_state_v9::state::ContractState, + ledger_v8::annotation::NightAnn, + ledger_v9::annotation::NightAnn, + >, + _, + >(PhantomData), + ), + ( + TranslationId( + Cow::Borrowed("mpt-node(contract-state[v6],night-annotation)"), + Cow::Borrowed("mpt-node(contract-state[v8],night-annotation)"), + ), + &DirectSpTranslation::< + merkle_patricia_trie::Node< + onchain_state_v8::state::ContractState, + D, + ledger_v8::annotation::NightAnn, + >, + merkle_patricia_trie::Node< + onchain_state_v9::state::ContractState, + D, + ledger_v9::annotation::NightAnn, + >, + MptTl< + onchain_state_v8::state::ContractState, + onchain_state_v9::state::ContractState, + ledger_v8::annotation::NightAnn, + ledger_v9::annotation::NightAnn, + >, + _, + >(PhantomData), + ), + // `bridge_receiving` MPT — entries unchanged (u128), annotation changes + // from SizeAnn to NightAnn. Needs an identity entry translation and an + // MptTl that re-annotates. + ( + TranslationId(Cow::Borrowed("u128"), Cow::Borrowed("u128")), + &DirectSpTranslation::, _>(PhantomData), + ), + ( + TranslationId( + Cow::Borrowed("mpt(u128,size-annotation)"), + Cow::Borrowed("mpt(u128,night-annotation)"), + ), + &DirectSpTranslation::< + MerklePatriciaTrie, + MerklePatriciaTrie, + MptTl, + _, + >(PhantomData), + ), + ( + TranslationId( + Cow::Borrowed("mpt-node(u128,size-annotation)"), + Cow::Borrowed("mpt-node(u128,night-annotation)"), + ), + &DirectSpTranslation::< + merkle_patricia_trie::Node, + merkle_patricia_trie::Node, + MptTl, + _, + >(PhantomData), + ), + ]; +} + +#[cfg(test)] +mod tests { + use super::*; + use storage::db::InMemoryDB; + + fn translate_to_completion( + v8: ledger_v8::structure::LedgerState, + ) -> ledger_v9::structure::LedgerState { + let tl_state = TypedTranslationState::< + ledger_v8::structure::LedgerState, + ledger_v9::structure::LedgerState, + StateTranslationTable, + InMemoryDB, + >::start(Sp::new(v8)) + .expect("Failed to start translation"); + + let cost = CostDuration::from_picoseconds(1_000_000_000_000); + let finished = tl_state.run(cost).expect("Translation failed"); + + finished + .result() + .expect("Failed to get result") + .expect("Translation did not complete") + .deref() + .clone() + } + + /// Every `TranslationId` a table entry requires must itself be in the table, + /// or translation errors at runtime the first time the entry is needed. + #[test] + fn table_is_closed() { + >::assert_closure(); + } + + /// The `TABLE` hardcodes tag string literals. If a tag on either the v8 or v9 + /// side drifts (e.g. an rc bump changes a `#[tag]`), the literal no longer + /// matches what `T::tag()` produces and the migration silently mis-dispatches. + /// Rebuild every expected ID from the node's actual crate types and compare. + #[test] + fn table_tags_match_types() { + use storage::merkle_patricia_trie::{MerklePatriciaTrie, Node}; + use storage::storable::SizeAnn; + + type V8Ann = ledger_v8::annotation::NightAnn; + type V9Ann = ledger_v9::annotation::NightAnn; + type V8Contract = onchain_state_v8::state::ContractState; + type V9Contract = onchain_state_v9::state::ContractState; + + let expected: Vec<(Cow<'static, str>, Cow<'static, str>)> = vec![ + ( + ledger_v8::structure::LedgerState::::tag(), + ledger_v9::structure::LedgerState::::tag(), + ), + ( + ledger_v8::structure::LedgerParameters::tag(), + ledger_v9::structure::LedgerParameters::tag(), + ), + (V8Contract::tag(), V9Contract::tag()), + ( + MerklePatriciaTrie::::tag(), + MerklePatriciaTrie::::tag(), + ), + ( + Node::::tag(), + Node::::tag(), + ), + (u128::tag(), u128::tag()), + ( + MerklePatriciaTrie::::tag(), + MerklePatriciaTrie::::tag(), + ), + (Node::::tag(), Node::::tag()), + ]; + + let actual: Vec<_> = >::TABLE + .iter() + .map(|(id, _)| (id.0.clone(), id.1.clone())) + .collect(); + + assert_eq!(actual, expected); + } + + /// End-to-end smoke test: a default v8 `LedgerState` translates to v9, + /// preserving the tag-stable pools and picking up the new v9 default + /// `min_block_price`, and survives a v9 serialize round-trip. + #[test] + fn empty_state_translates_and_round_trips() { + let v8 = ledger_v8::structure::LedgerState::::new("test-network"); + let v9 = translate_to_completion(v8.clone()); + + assert_eq!(v9.network_id, v8.network_id); + assert_eq!(v9.reserve_pool, v8.reserve_pool); + assert_eq!(v9.locked_pool, v8.locked_pool); + assert_eq!(v9.block_reward_pool, v8.block_reward_pool); + assert_eq!( + v9.parameters.min_block_price, + ledger_v9::structure::INITIAL_PARAMETERS.min_block_price, + ); + + let mut buf = Vec::new(); + serialize::tagged_serialize(&v9, &mut buf).expect("v9 serialize"); + let v9_rt: ledger_v9::structure::LedgerState = + serialize::tagged_deserialize(&mut &buf[..]).expect("v9 deserialize"); + assert_eq!(v9_rt.network_id, v9.network_id); + } +} diff --git a/ledger/src/host_api/ledger_8.rs b/ledger/src/host_api/ledger_8.rs index 3222f875e..de03ac54e 100644 --- a/ledger/src/host_api/ledger_8.rs +++ b/ledger/src/host_api/ledger_8.rs @@ -429,6 +429,20 @@ pub trait Ledger8Bridge { } } + /// The ledger-8 runtime imports this to pay block rewards to the treasury. + /// Retained (removed for v9) so the current node can execute the ledger-8 + /// WASM across the 8->9 hardfork boundary. + fn construct_distribute_treasury_system_tx( + &mut self, + amount: PassFatPointerAndDecode, + ) -> AllocateAndReturnByCodec, LedgerApiError>> { + if is_unified(*self) { + Bridge::::construct_distribute_treasury_system_tx(amount) + } else { + Bridge::::construct_distribute_treasury_system_tx(amount) + } + } + /// Ensures the correct ledger storage is initialized for this runtime version. /// Handles rollback: if new version's storage is initialized but we need this version's storage, /// drops new version's storage and initializes normal storage. diff --git a/ledger/src/host_api/ledger_9.rs b/ledger/src/host_api/ledger_9.rs index 53f234c4c..982082939 100644 --- a/ledger/src/host_api/ledger_9.rs +++ b/ledger/src/host_api/ledger_9.rs @@ -45,6 +45,52 @@ fn is_unified(mut ext: &mut dyn Externalities) -> bool { ) } +#[cfg(feature = "std")] +use crate::ledger_8::Bridge as Bridge8; +#[cfg(feature = "std")] +type Signature8 = crate::ledger_8::TransactionSignature; + +/// Translate a ledger-8 `LedgerApiError` into its ledger-9 counterpart. +/// +/// The two are distinct types generated from the same source +/// (`versions/common/types.rs`) by module parameterization, so their SCALE +/// encodings are identical by construction. Round-tripping keeps this correct +/// when a variant is added, where a hand-written match would need editing in +/// lockstep. `ledger_8_error_encoding_matches_ledger_9` guards the assumption. +#[cfg(feature = "std")] +fn as_ledger_9_error(error: crate::ledger_8::types::LedgerApiError) -> LedgerApiError { + use parity_scale_codec::{Decode, Encode}; + LedgerApiError::decode(&mut &error.encode()[..]).unwrap_or(LedgerApiError::HostApiError) +} + +/// Serve a read-only ledger accessor from the ledger-8 bridge when `$state_key` +/// is a ledger-8 arena root, by returning early from the enclosing host function. +/// Falls through to the ledger-9 body otherwise. +/// +/// For the ledger 8 -> 9 hard-fork, `system_version == 1` which means runtime code +/// is applied during the upgrade block rather than queued to be applied in the next +/// block. This code allows off-chain runtime calls to access historic block data +/// using the correct ledger api despite the runtime code/chain data skew. +/// +/// This will not be needed for future forks; see: +/// - https://github.com/midnightntwrk/midnight-node/pull/1900 +/// +/// `$call` names the `Bridge` method and takes its arguments verbatim; only the +/// storage-mode dispatch and the error translation are supplied here. +#[cfg(feature = "std")] +macro_rules! serve_pre_migration_v8_read { + ($ext:expr, $state_key:expr, $call:ident($($arg:expr),* $(,)?)) => { + if crate::is_ledger_8_state_key($state_key) { + let result = if is_unified($ext) { + Bridge8::::$call($($arg),*) + } else { + Bridge8::::$call($($arg),*) + }; + return result.map_err(as_ledger_9_error); + } + }; +} + #[runtime_interface] pub trait Ledger9Bridge { fn set_default_storage(&mut self) { @@ -219,6 +265,12 @@ pub trait Ledger9Bridge { state_key: PassFatPointerAndRead<&[u8]>, contract_address: PassFatPointerAndRead<&[u8]>, ) -> AllocateAndReturnByCodec, LedgerApiError>> { + serve_pre_migration_v8_read!( + *self, + state_key, + get_contract_state(state_key, contract_address) + ); + if is_unified(*self) { Bridge::::get_contract_state(state_key, contract_address) } else { @@ -250,6 +302,12 @@ pub trait Ledger9Bridge { state_key: PassFatPointerAndRead<&[u8]>, contract_address: PassFatPointerAndRead<&[u8]>, ) -> AllocateAndReturnByCodec, LedgerApiError>> { + serve_pre_migration_v8_read!( + *self, + state_key, + get_zswap_chain_state(state_key, contract_address) + ); + if is_unified(*self) { Bridge::::get_zswap_chain_state(state_key, contract_address) } else { @@ -266,6 +324,12 @@ pub trait Ledger9Bridge { state_key: PassFatPointerAndRead<&[u8]>, beneficiary: PassFatPointerAndRead<&[u8]>, ) -> AllocateAndReturnByCodec> { + serve_pre_migration_v8_read!( + *self, + state_key, + get_unclaimed_amount(state_key, beneficiary) + ); + if is_unified(*self) { Bridge::::get_unclaimed_amount(state_key, beneficiary) } else { @@ -281,6 +345,8 @@ pub trait Ledger9Bridge { &mut self, state_key: PassFatPointerAndRead<&[u8]>, ) -> AllocateAndReturnByCodec, LedgerApiError>> { + serve_pre_migration_v8_read!(*self, state_key, get_ledger_parameters(state_key)); + if is_unified(*self) { Bridge::::get_ledger_parameters(state_key) } else { @@ -296,6 +362,8 @@ pub trait Ledger9Bridge { &mut self, state_key: PassFatPointerAndRead<&[u8]>, ) -> AllocateAndReturnByCodec> { + serve_pre_migration_v8_read!(*self, state_key, get_c_to_m_bridge_min_amount(state_key)); + if is_unified(*self) { Bridge::::get_c_to_m_bridge_min_amount(state_key) } else { @@ -305,6 +373,14 @@ pub trait Ledger9Bridge { /* * Returns the expected fee to pay for a submitting a transaction + * + * No `serve_pre_migration_v8_read!` guard here, unlike the accessors above: a + * cost estimate is always requested for a transaction about to be submitted, + * and `get_ledger_version` reports ledger 9 as soon as the new code is live, so + * `tx` is a v9-format transaction that ledger-8 code cannot deserialize anyway. + * The same reasoning covers the transaction paths (`validate_transaction`, + * `apply_transaction`, ...): at the skew block they concern v9 transactions, and + * they resolve on their own one block later once the migration has run. */ fn get_transaction_cost( &mut self, @@ -338,6 +414,8 @@ pub trait Ledger9Bridge { &mut self, state_key: PassFatPointerAndRead<&[u8]>, ) -> AllocateAndReturnByCodec, LedgerApiError>> { + serve_pre_migration_v8_read!(*self, state_key, get_zswap_state_root(state_key)); + if is_unified(*self) { Bridge::::get_zswap_state_root(state_key) } else { @@ -360,6 +438,8 @@ pub trait Ledger9Bridge { &mut self, state_key: PassFatPointerAndRead<&[u8]>, ) -> AllocateAndReturnByCodec, LedgerApiError>> { + serve_pre_migration_v8_read!(*self, state_key, get_ledger_state_root(state_key)); + if is_unified(*self) { Bridge::::get_ledger_state_root(state_key) } else { @@ -469,6 +549,32 @@ pub trait Ledger9Bridge { true } + /// Translate the ledger state from ledger-v8 format to ledger-v9 format. + /// + /// Called by `pallet_midnight`'s v8->v9 storage migration during the runtime + /// upgrade that crosses into ledger-9. `state_key` is the pallet's `StateKey` + /// (a v8 arena root); returns the new v9 arena root to store back, together + /// with the synthetic cost (picoseconds) the translation consumed against + /// the ledger's cost model, for the pallet to charge as this migration's + /// weight. + fn migrate_state_v8_to_v9( + &mut self, + state_key: PassFatPointerAndRead<&[u8]>, + ) -> AllocateAndReturnByCodec, u64), LedgerApiError>> { + // Ensure the ledger arena is initialized before translating. The migration + // runs in the Executive migrations tuple, before pallet_midnight's + // on_initialize/on_runtime_upgrade have (re)initialized storage this block. + // `set_default_storage` is idempotent — a no-op if the pre-fork ledger-8 + // blocks already set it (v8 and v9 share the same storage backend). + if is_unified(*self) { + Bridge::::set_default_storage(*self); + crate::host_api::migration_8_to_9::migrate_state_v8_to_v9::(state_key) + } else { + Bridge::::set_default_storage(*self); + crate::host_api::migration_8_to_9::migrate_state_v8_to_v9::(state_key) + } + } + /// Initialize a process-wide temporary ledger ParityDb seeded with the /// undeployed-network genesis state. /// @@ -502,3 +608,36 @@ pub trait Ledger9Bridge { }); } } + +#[cfg(all(test, feature = "std"))] +mod tests { + use super::as_ledger_9_error; + use crate::{ledger_8::types as v8, ledger_9::types as v9}; + + /// `as_ledger_9_error` relies on the two versions' `LedgerApiError` sharing a + /// SCALE encoding, which holds because both are generated from + /// `versions/common/types.rs`. Pin that down — including a nested payload and + /// the last variant, which is where a divergence would first show up — so a + /// future edit to one version's enum fails here rather than silently turning + /// every pre-migration read error into `HostApiError`. + #[test] + fn ledger_8_error_encoding_matches_ledger_9() { + let cases = [ + (v8::LedgerApiError::NoLedgerState, v9::LedgerApiError::NoLedgerState), + (v8::LedgerApiError::ContractNotPresent, v9::LedgerApiError::ContractNotPresent), + (v8::LedgerApiError::BeneficiaryNotFound, v9::LedgerApiError::BeneficiaryNotFound), + ( + v8::LedgerApiError::Deserialization(v8::DeserializationError::TypedArenaKey), + v9::LedgerApiError::Deserialization(v9::DeserializationError::TypedArenaKey), + ), + ( + v8::LedgerApiError::Serialization(v8::SerializationError::LedgerParameters), + v9::LedgerApiError::Serialization(v9::SerializationError::LedgerParameters), + ), + ]; + + for (from, expected) in cases { + assert_eq!(as_ledger_9_error(from.clone()), expected, "mistranslated {from:?}"); + } + } +} diff --git a/ledger/src/host_api/migration_8_to_9.rs b/ledger/src/host_api/migration_8_to_9.rs new file mode 100644 index 000000000..6fe05c873 --- /dev/null +++ b/ledger/src/host_api/migration_8_to_9.rs @@ -0,0 +1,222 @@ +// This file is part of midnight-node. +// Copyright (C) 2025-2026 Midnight Foundation +// SPDX-License-Identifier: Apache-2.0 +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Host-side v8 -> v9 ledger state translation, driven by +//! [`crate::state_translation_v8_to_v9::StateTranslationTable`]. +//! +//! The on-chain pallet stores only the arena root of the ledger state +//! (`pallet_midnight::StateKey`, a `tagged_serialize`d `TypedArenaKey`); +//! the `LedgerState` itself lives in the process-global ledger arena (parity-db). +//! When the runtime upgrades from a ledger-8 runtime (spec < 2_000_000) to a +//! ledger-9 runtime (spec >= 2_000_000), the on-chain migration +//! (`pallet_midnight::migrations`) calls [`Ledger9Bridge::migrate_state_v8_to_v9`] +//! which lands here: it reads the v8 arena root, walks the v8 `LedgerState` +//! translating it into a v9 `LedgerState`, re-persists it, and returns the new v9 +//! arena root for the pallet to store back into `StateKey`. +//! +//! v8 and v9 share one storage crate (`ledger-storage-ledger-8`, +//! midnight-storage 2.0.1) and hence one arena, so the translation reads and +//! writes the same parity-db instance the pre-fork ledger-8 blocks populated. + +use crate::ledger_8::api::Ledger as Ledger8; +use crate::ledger_9::api::Ledger as Ledger9; +use crate::ledger_9::types::{DeserializationError, LedgerApiError, SerializationError}; +use midnight_node_ledger_helpers::state_translation_v8_to_v9::StateTranslationTable; + +use base_crypto::cost_model::CostDuration; +use ledger_storage_ledger_8 as storage; +use midnight_serialize::{tagged_deserialize, tagged_serialize}; +use storage::{ + arena::{Sp, TypedArenaKey}, + db::DB, + state_translation::TypedTranslationState, + storage::default_storage, +}; + +type LedgerState8 = mn_ledger_8::structure::LedgerState; +type LedgerState9 = mn_ledger_9::structure::LedgerState; + +const LOG_TARGET: &str = "midnight::ledger::migration_8_to_9"; + +/// Picoseconds granted to each `TypedTranslationState::run` step. The migration +/// is single-block (it must complete within one host call), so we loop over +/// `run` with a per-step budget until the translation reports a result. +/// +/// This budget also doubles as the quantization granularity of the synthetic +/// cost reported back to the pallet (see `consumed_cost_ps` below): `run` +/// doesn't return unused budget, so every completed step is charged in full. +/// 10ms keeps that over-approximation small relative to real translation +/// costs while still comfortably draining dev/undeployed-sized state in a +/// handful of iterations; the loop cap is a runaway backstop only. +const RUN_BUDGET_PS: u64 = 10_000_000_000; +const MAX_STEPS: usize = 100_000; + +/// Translate the ledger state referenced by a v8 arena root (`state_key_v8`, +/// the pallet's `StateKey` bytes) into a v9 ledger state, persist it, and +/// return the new v9 arena root (to store back into `StateKey`) together with +/// the synthetic cost, in picoseconds, the translation consumed against the +/// ledger's deterministic cost model — for the pallet to charge as this +/// migration's weight. +/// +/// If `state_key_v8` already references a ledger-9 state this is a no-op: it +/// returns the key unchanged with zero cost (see the idempotency guard below). +pub fn migrate_state_v8_to_v9( + state_key_v8: &[u8], +) -> Result<(Vec, u64), LedgerApiError> { + let t_total = std::time::Instant::now(); + + // 0. Idempotency guard: no-op if the state is already ledger-9. + // + // The pallet gates this behind `VersionedMigration<1, 2>`, but the on-chain + // pallet-midnight storage version is not a faithful proxy for the ledger + // version. The 2.0.0 runtime (spec 2_000_000) already runs ledger-9 yet + // shipped pallet-midnight at storage version 1 (it had no v1->v2 migration), + // so a network upgrading 2.0.0 -> this runtime still triggers this migration + // even though its `StateKey` already points at a v9 arena root. Feeding that + // v9 root to the v8 decode below would fail on the tag mismatch and abort the + // upgrade (the pallet `expect`s success), bricking the chain. Detect it by + // the root's serialized tag — `TypedArenaKey`'s tag embeds the `LedgerState` + // version (`storage-key(midnight:ledger-state[vN]:...)`), so a successful + // tagged decode as a v9 key is a reliable, arena-free discriminator — and + // return the key unchanged with zero synthetic cost. + if tagged_deserialize::, D::Hasher>>(&mut &state_key_v8[..]).is_ok() { + log::info!( + target: LOG_TARGET, + "StateKey already references a ledger-9 state; skipping v8->v9 translation (no-op)" + ); + return Ok((state_key_v8.to_vec(), 0)); + } + + // 1. Decode the v8 arena root and load the v8 ledger wrapper from the arena. + let t_load = std::time::Instant::now(); + let key8: TypedArenaKey, D::Hasher> = tagged_deserialize(&mut &state_key_v8[..]) + .map_err(|e| { + log::error!(target: LOG_TARGET, "failed to deserialize v8 state key: {e:?}"); + LedgerApiError::Deserialization(DeserializationError::TypedArenaKey) + })?; + let ledger8: Sp, D> = default_storage::().arena.get_lazy(&key8).map_err(|e| { + log::error!(target: LOG_TARGET, "failed to load v8 ledger from arena: {e:?}"); + LedgerApiError::NoLedgerState + })?; + log::debug!(target: LOG_TARGET, "[perf] migrate_state_v8_to_v9 load took {:?}", t_load.elapsed()); + + // 2. Run the state translation table over the inner v8 `LedgerState`. + let t_translate = std::time::Instant::now(); + let input: Sp, D> = Sp::new(ledger8.state.clone()); + let mut tl = + TypedTranslationState::, LedgerState9, StateTranslationTable, D>::start( + input, + ) + .map_err(|e| { + log::error!(target: LOG_TARGET, "failed to start v8->v9 translation: {e:?}"); + LedgerApiError::HostApiError + })?; + + let run_budget = CostDuration::from_picoseconds(RUN_BUDGET_PS); + let mut steps = 0usize; + let state9: Sp, D> = loop { + steps += 1; + if steps > MAX_STEPS { + log::error!(target: LOG_TARGET, "v8->v9 translation did not converge in {MAX_STEPS} steps"); + return Err(LedgerApiError::HostApiError); + } + tl = tl.run(run_budget).map_err(|e| { + log::error!(target: LOG_TARGET, "v8->v9 translation step failed: {e:?}"); + LedgerApiError::HostApiError + })?; + if let Some(result) = tl.result().map_err(|e| { + log::error!(target: LOG_TARGET, "v8->v9 translation result failed: {e:?}"); + LedgerApiError::HostApiError + })? { + break result; + } + }; + // Every completed `run` call before the last one must have exhausted its + // full `RUN_BUDGET_PS` — otherwise `tl.result()` would already have been + // `Some` and the loop would have stopped there. Only the final call may + // have used less. `steps * RUN_BUDGET_PS` is therefore a deterministic + // upper bound on the synthetic cost actually consumed, accurate to one + // `RUN_BUDGET_PS` quantum. + let consumed_cost_ps = steps as u64 * RUN_BUDGET_PS; + log::info!( + target: LOG_TARGET, + "v8->v9 ledger state translation complete in {steps} step(s), {:?}, {consumed_cost_ps}ps synthetic cost", + t_translate.elapsed() + ); + + // 3. Wrap the translated state in the v9 ledger wrapper, persist it, and + // flush the arena so the new root is durable before the pallet stores it. + let t_persist = std::time::Instant::now(); + let ledger9 = Ledger9::new((*state9).clone()); + let mut sp9: Sp, D> = default_storage::().arena.alloc(ledger9); + sp9.persist(); + default_storage::().with_backend(|backend| backend.flush_all_changes_to_db()); + log::debug!( + target: LOG_TARGET, + "[perf] migrate_state_v8_to_v9 persist+flush took {:?}", + t_persist.elapsed() + ); + + // 4. Serialize the new v9 arena root for the pallet to store in `StateKey`. + let mut bytes = Vec::new(); + tagged_serialize(&sp9.as_typed_key(), &mut bytes).map_err(|e| { + log::error!(target: LOG_TARGET, "failed to serialize v9 state key: {e:?}"); + LedgerApiError::Serialization(SerializationError::TypedArenaKey) + })?; + + log::debug!(target: LOG_TARGET, "[perf] migrate_state_v8_to_v9 took {:?}", t_total.elapsed()); + Ok((bytes, consumed_cost_ps)) +} + +#[cfg(test)] +mod tests { + use super::*; + use ledger_storage_ledger_8::db::InMemoryDB; + + /// Dev-only: verify that seeding a v8 genesis blob with *this* crate's + /// ledger_8 `Ledger` wrapper reproduces the exact arena root a ledger-8 node + /// (e.g. release 1.0.1) stored in the chain-spec as `genesisStateKey`. If the + /// wrapper serialization drifted, the seeded root wouldn't match and block + /// execution after boot would fail to find the state. Runs only when the two + /// blob paths are provided via env (extracted from a fork-from chain-spec). + #[test] + fn v8_genesis_seed_root_matches_chainspec_key() { + let (Ok(gs_path), Ok(key_path)) = + (std::env::var("HF_GENESIS_STATE"), std::env::var("HF_GENESIS_KEY")) + else { + eprintln!("skipping: set HF_GENESIS_STATE and HF_GENESIS_KEY to run"); + return; + }; + let genesis = std::fs::read(gs_path).expect("read genesis_state"); + let expected = std::fs::read(key_path).expect("read genesisStateKey"); + + let state: LedgerState8 = + midnight_serialize::tagged_deserialize(&mut &genesis[..]) + .expect("deserialize v8 state"); + let ledger = Ledger8::::new(state); + let mut sp = default_storage::().arena.alloc(ledger); + sp.persist(); + let mut got = Vec::new(); + tagged_serialize(&sp.as_typed_key(), &mut got).expect("serialize key"); + + assert_eq!( + got, + expected, + "seeded v8 root must match the chain-spec genesisStateKey \n got={} \n exp={}", + hex::encode(&got), + hex::encode(&expected), + ); + } +} diff --git a/ledger/src/host_api/mod.rs b/ledger/src/host_api/mod.rs index 2b60cdd3a..1b1fd44c3 100644 --- a/ledger/src/host_api/mod.rs +++ b/ledger/src/host_api/mod.rs @@ -14,3 +14,7 @@ pub mod ledger_7; pub mod ledger_8; pub mod ledger_9; + +/// Host-side v8 -> v9 ledger state translation used by the runtime storage migration. +#[cfg(feature = "std")] +pub mod migration_8_to_9; diff --git a/ledger/src/lib.rs b/ledger/src/lib.rs index 1c65c92db..1ad985a36 100644 --- a/ledger/src/lib.rs +++ b/ledger/src/lib.rs @@ -145,6 +145,71 @@ pub fn drop_all_default_storage() { ledger_9::storage::drop_default_storage_if_exists(); } +/// Seed the (separate) ledger arena from a genesis `LedgerState` blob, using the +/// deserializer that matches the blob's `ledger-state[vN]` header tag. +/// +/// A node may boot on a chain-spec produced by an older runtime — notably the +/// ledger 8->9 hardfork, where a ledger-9 node starts from a ledger-8 +/// (`ledger-state[v13]`) genesis and only upgrades to v9 later via the runtime +/// migration. Seeding must therefore match the genesis version (the genesis +/// block runs under the old WASM and expects the old-format arena root), not the +/// latest. v8 and v9 share one storage backend, so a v8-seeded arena is exactly +/// what the post-migration v9 runtime reads. Unrecognized tags fall back to the +/// latest version (`ledger_9`), preserving the prior default behaviour. +#[cfg(feature = "std")] +pub fn init_ledger_storage_separate>( + dir: P, + genesis_state: &[u8], + cache_size: usize, +) -> alloc::vec::Vec { + if ledger_8::storage::genesis_matches_this_version(genesis_state) { + ledger_8::storage::init_storage_paritydb_separate(dir, genesis_state, cache_size) + } else { + ledger_9::storage::init_storage_paritydb_separate(dir, genesis_state, cache_size) + } +} + +/// Unified-DB counterpart of [`init_ledger_storage_separate`]. +#[cfg(feature = "std")] +pub fn init_ledger_storage_unified< + D: core::ops::Deref + Default + Send + Sync + 'static, + const COLUMN_OFFSET: u8, +>( + db_instance: D, + genesis_state: &[u8], + cache_size: usize, +) -> alloc::vec::Vec { + if ledger_8::storage::genesis_matches_this_version(genesis_state) { + ledger_8::storage::init_storage_paritydb_unified::( + db_instance, + genesis_state, + cache_size, + ) + } else { + ledger_9::storage::init_storage_paritydb_unified::( + db_instance, + genesis_state, + cache_size, + ) + } +} + +/// Returns true if `state_key` is a ledger-8 arena root, i.e. a tagged-serialized +/// `TypedArenaKey, _>`. +#[cfg(feature = "std")] +pub(crate) fn is_ledger_8_state_key(state_key: &[u8]) -> bool { + use ledger_storage_ledger_8::{DefaultDB, arena::TypedArenaKey, db::DB}; + use midnight_serialize::Tagged; + + type Ledger8Root = TypedArenaKey, ::Hasher>; + + let expected = ::tag(); + match midnight_serialize::peek_tag(&mut std::io::Cursor::new(state_key)) { + Ok(tag) => tag.as_str() == expected.as_ref(), + Err(_) => false, + } +} + mod common; pub mod types { @@ -188,4 +253,32 @@ mod tests { unsafe_drop_default_storage::(); assert!(try_get_default_storage::().is_none()); } + + /// `is_ledger_8_state_key` is what the ledger-9 host API dispatches on to read the + /// `set_code` block of the 8->9 hardfork, whose `StateKey` is one version behind + /// its `:code` (GH #1959). It has to tell a ledger-8 arena root from a ledger-9 + /// one from the header tag alone. + #[test] + fn ledger_8_state_key_tag_is_recognised() { + use ledger_storage_ledger_8::DefaultDB; + use midnight_serialize::{GLOBAL_TAG, Tagged}; + + // A `StateKey` is `tagged_serialize(&Sp, D>::as_typed_key())`, and + // `TypedArenaKey`'s tag wraps its referent's — which for `Ledger` is just + // `LedgerState`'s. Only the header matters here; `peek_tag` never reads the body. + fn header() -> Vec { + format!("{GLOBAL_TAG}storage-key({}):", T::tag()).into_bytes() + } + let v8 = header::>(); + let v9 = header::>(); + assert_ne!(v8, v9, "v8 and v9 ledger states must not share a tag"); + + assert!(super::is_ledger_8_state_key(&v8)); + assert!(!super::is_ledger_8_state_key(&v9)); + + // An unset `StateKey`, or anything else untagged, is not a ledger-8 root: the + // host API must take its ordinary ledger-9 path rather than guess. + assert!(!super::is_ledger_8_state_key(&[])); + assert!(!super::is_ledger_8_state_key(b"not-tagged-at-all")); + } } diff --git a/ledger/src/versions/common/mod.rs b/ledger/src/versions/common/mod.rs index 1d2bc61e4..5aca07933 100644 --- a/ledger/src/versions/common/mod.rs +++ b/ledger/src/versions/common/mod.rs @@ -1115,6 +1115,14 @@ where let system_tx = super::system_tx::unlock_to_treasury_system_tx(amount)?; api.tagged_serialize(&system_tx) } + + pub fn construct_distribute_treasury_system_tx( + amount: u128, + ) -> Result, LedgerApiError> { + let api = api::new(); + let system_tx = super::system_tx::distribute_treasury_system_tx(amount)?; + api.tagged_serialize(&system_tx) + } } #[cfg(feature = "std")] diff --git a/ledger/src/versions/common/storage.rs b/ledger/src/versions/common/storage.rs index ef39ea8d9..496757b52 100644 --- a/ledger/src/versions/common/storage.rs +++ b/ledger/src/versions/common/storage.rs @@ -74,6 +74,25 @@ impl core::fmt::Display for GetRootError { } } +/// Returns true if `genesis_state` is a tagged-serialized `LedgerState` of *this* +/// ledger version (i.e. its `midnight:ledger-state[vN]:` header tag matches this +/// version's `LedgerState::tag()`). +/// +/// Used to pick the correct version-specific seeder for the genesis arena when a +/// node boots on a chain-spec produced by an older runtime (e.g. the ledger 8->9 +/// hardfork, where a ledger-9 node starts from a ledger-8 `ledger-state[v13]` +/// genesis before the runtime upgrade migrates it to v9). +#[cfg(feature = "std")] +pub fn genesis_matches_this_version(genesis_state: &[u8]) -> bool { + use super::ledger_storage_local::DefaultDB; + let expected = as Tagged>::tag(); + // `peek_tag` reads the serialized header tag without deserializing the body. + match super::midnight_serialize_local::peek_tag(&mut std::io::Cursor::new(genesis_state)) { + Ok(tag) => tag.as_str() == expected.as_ref(), + Err(_) => false, + } +} + pub fn get_root(state: &[u8], network_id: Option<&str>) -> Result, GetRootError> { // Get empty state key use super::api::Ledger; diff --git a/ledger/src/versions/system_tx/ledger_7.rs b/ledger/src/versions/system_tx/ledger_7.rs index d6c5807c3..292832b0d 100644 --- a/ledger/src/versions/system_tx/ledger_7.rs +++ b/ledger/src/versions/system_tx/ledger_7.rs @@ -37,3 +37,8 @@ pub fn unlock_to_treasury_system_tx(_amount: u128) -> Result bool { false } + +/// Not applicable to ledger-7 (only the ledger-8 bridge exposes this host fn). +pub fn distribute_treasury_system_tx(_amount: u128) -> Result { + Err(LedgerApiError::HostApiError) +} diff --git a/ledger/src/versions/system_tx/ledger_8.rs b/ledger/src/versions/system_tx/ledger_8.rs index d6c5807c3..a893345eb 100644 --- a/ledger/src/versions/system_tx/ledger_8.rs +++ b/ledger/src/versions/system_tx/ledger_8.rs @@ -34,6 +34,14 @@ pub fn unlock_to_treasury_system_tx(_amount: u128) -> Result9 hardfork boundary — the ledger-8 runtime imports the corresponding +/// `construct_distribute_treasury_system_tx` host function (removed for v9). +pub fn distribute_treasury_system_tx(amount: u128) -> Result { + Ok(SystemTransaction::PayBlockRewardsToTreasury { amount }) +} + pub fn is_unlock_to_treasury_system_tx(_tx: &SystemTransaction) -> bool { false } diff --git a/ledger/src/versions/system_tx/ledger_9.rs b/ledger/src/versions/system_tx/ledger_9.rs index 6e854f4a3..99bd7a762 100644 --- a/ledger/src/versions/system_tx/ledger_9.rs +++ b/ledger/src/versions/system_tx/ledger_9.rs @@ -32,3 +32,10 @@ pub fn unlock_to_treasury_system_tx(amount: u128) -> Result bool { matches!(tx, SystemTransaction::UnlockToTreasury { .. }) } + +/// Not applicable to ledger-9: the block-rewards-to-treasury system tx was +/// removed for v9 (only the ledger-8 bridge exposes this host fn, so the ledger-8 +/// WASM can be executed across the 8->9 hardfork). +pub fn distribute_treasury_system_tx(_amount: u128) -> Result { + Err(LedgerApiError::HostApiError) +} diff --git a/local-environment/package-lock.json b/local-environment/package-lock.json index 57fd4af71..b7b36a78d 100644 --- a/local-environment/package-lock.json +++ b/local-environment/package-lock.json @@ -1450,6 +1450,18 @@ "node": ">=0.4.0" } }, + "node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "license": "MIT", + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, "node_modules/ajv": { "version": "6.14.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.14.0.tgz", @@ -1513,13 +1525,14 @@ "license": "MIT" }, "node_modules/axios": { - "version": "1.16.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.16.0.tgz", - "integrity": "sha512-6hp5CwvTPlN2A31g5dxnwAX0orzM7pmCRDLnZSX772mv8WDqICwFjowHuPs04Mc8deIld1+ejhtaMn5vp6b+1w==", + "version": "1.19.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.19.0.tgz", + "integrity": "sha512-ht/iuYZXEjFxLH/Hkezgd7m6JKlHHXEUSneaDz8uZe1Gj5QZtCnpyDsckvAiEnT89OEbCLmnte4R4sn7P0EKFw==", "license": "MIT", "dependencies": { "follow-redirects": "^1.16.0", - "form-data": "^4.0.5", + "form-data": "^4.0.6", + "https-proxy-agent": "^5.0.1", "proxy-from-env": "^2.1.0" } }, @@ -1539,15 +1552,15 @@ "license": "MIT" }, "node_modules/brace-expansion": { - "version": "5.0.6", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.6.tgz", - "integrity": "sha512-kLpxurY4Z4r9sgMsyG0Z9uzsBlgiU/EFKhj/h91/8yHu0edo7XuixOIH3VcJ8kkxs6/jPzoI6U9Vj3WqbMQ94g==", + "version": "5.0.9", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz", + "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==", "license": "MIT", "dependencies": { "balanced-match": "^4.0.2" }, "engines": { - "node": "18 || 20 || >=22" + "node": "20 || >=22" } }, "node_modules/call-bind-apply-helpers": { @@ -2268,6 +2281,19 @@ "node": ">= 0.4" } }, + "node_modules/https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "license": "MIT", + "dependencies": { + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/ignore": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", @@ -2372,9 +2398,9 @@ } }, "node_modules/js-yaml": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.2.0.tgz", - "integrity": "sha512-ePWsvanv0DWuDRsW8dnt+R4jQ31SCRCQ7hhNcPXZPsoBZiemuZNYGf7adZdqX2D86j6rvKp3RpCxVTSb8WQlOw==", + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.1.tgz", + "integrity": "sha512-CY6crGq313MX8GkwvB7tzgp99vjQxY1++5y10/BKN/GUfHqWaOGQMNZkBvqSzsZKWk/ijwHlWzzkLulsGHhjWQ==", "funding": [ { "type": "github", diff --git a/metadata/Cargo.toml b/metadata/Cargo.toml index 627859ed4..635b0e763 100644 --- a/metadata/Cargo.toml +++ b/metadata/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "midnight-node-metadata" -version = "2.0.0" +version = "2.1.0" edition = "2024" build = "build.rs" license-file.workspace = true diff --git a/metadata/src/lib.rs b/metadata/src/lib.rs index 4c59f48a4..30a73e909 100644 --- a/metadata/src/lib.rs +++ b/metadata/src/lib.rs @@ -11,4 +11,7 @@ pub mod midnight_metadata_1_0_0 {} #[subxt::subxt(runtime_metadata_path = "static/midnight_metadata_2.0.0.scale")] pub mod midnight_metadata_2_0_0 {} -pub use midnight_metadata_2_0_0 as midnight_metadata_latest; +#[subxt::subxt(runtime_metadata_path = "static/midnight_metadata_2.1.0.scale")] +pub mod midnight_metadata_2_1_0 {} + +pub use midnight_metadata_2_1_0 as midnight_metadata_latest; diff --git a/metadata/static/midnight_metadata.scale b/metadata/static/midnight_metadata.scale index 7c4ab497c..619fd321e 100644 Binary files a/metadata/static/midnight_metadata.scale and b/metadata/static/midnight_metadata.scale differ diff --git a/metadata/static/midnight_metadata_2.1.0.scale b/metadata/static/midnight_metadata_2.1.0.scale new file mode 100644 index 000000000..619fd321e Binary files /dev/null and b/metadata/static/midnight_metadata_2.1.0.scale differ diff --git a/node/Cargo.toml b/node/Cargo.toml index 0f9a52fa0..228e5a025 100644 --- a/node/Cargo.toml +++ b/node/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "midnight-node" -version = "2.0.0" +version = "2.1.0" description = "Midnight blockchain node" authors = ["Substrate DevHub "] homepage = "https://substrate.io/" diff --git a/node/src/backend/custom_parity_db.rs b/node/src/backend/custom_parity_db.rs index 5cd5ab729..ea4693256 100644 --- a/node/src/backend/custom_parity_db.rs +++ b/node/src/backend/custom_parity_db.rs @@ -109,7 +109,10 @@ pub fn open>( match storage_config.separation { StorageSeparation::Separate => { - midnight_node_ledger::ledger_9::storage::init_storage_paritydb_separate( + // Version-aware: a ledger-9 node may boot on a ledger-8 genesis during + // the 8->9 hardfork; seed the arena with the deserializer matching the + // genesis `ledger-state[vN]` tag (see `init_ledger_storage_separate`). + midnight_node_ledger::init_ledger_storage_separate( &storage_config.db_path, &storage_config.genesis_state, storage_config.cache_size, @@ -117,10 +120,11 @@ pub fn open>( Ok((OwnedDb(db), LedgerStorageDb::SeparateDb(storage_config.db_path.clone()))) }, StorageSeparation::Unified => { - midnight_node_ledger::ledger_9::storage::init_storage_paritydb_unified::< - _, - NUM_COLUMNS_POLKADOT, - >(OwnedDb(db.clone()), &storage_config.genesis_state, storage_config.cache_size); + midnight_node_ledger::init_ledger_storage_unified::<_, NUM_COLUMNS_POLKADOT>( + OwnedDb(db.clone()), + &storage_config.genesis_state, + storage_config.cache_size, + ); Ok((OwnedDb(db.clone()), LedgerStorageDb::UnifiedDb(db.clone()))) }, } diff --git a/pallets/midnight/src/lib.rs b/pallets/midnight/src/lib.rs index a829d4ce0..8a6c36225 100644 --- a/pallets/midnight/src/lib.rs +++ b/pallets/midnight/src/lib.rs @@ -99,7 +99,10 @@ pub mod pallet { } } - const STORAGE_VERSION: StorageVersion = StorageVersion::new(1); + // v2: ledger v8 -> v9 state translation (see `migrations::v2`). A ledger-8 + // runtime is at on-chain version 1; upgrading to this runtime runs the + // `MigrateV1ToV2` translation. Fresh ledger-9 genesis starts at version 2. + const STORAGE_VERSION: StorageVersion = StorageVersion::new(2); // Manually add ~1% of block weight pub const EXTRA_WEIGHT_TX_SIZE: Weight = Weight::from_parts(20_000_000_000, 0); diff --git a/pallets/midnight/src/migrations/mod.rs b/pallets/midnight/src/migrations/mod.rs index bc3f4785d..c7a82f25e 100644 --- a/pallets/midnight/src/migrations/mod.rs +++ b/pallets/midnight/src/migrations/mod.rs @@ -23,3 +23,6 @@ pub const PALLET_MIGRATIONS_ID: &[u8; 19] = b"pallet-midnight-mbm"; // See https://github.com/input-output-hk/midnight-substrate-prototype/pull/382 // for the example of such a migration. // pub mod v1; + +/// Single-block ledger v8 -> v9 state translation (storage version 1 -> 2). +pub mod v2; diff --git a/pallets/midnight/src/migrations/v2.rs b/pallets/midnight/src/migrations/v2.rs new file mode 100644 index 000000000..e9154eca7 --- /dev/null +++ b/pallets/midnight/src/migrations/v2.rs @@ -0,0 +1,117 @@ +// This file is part of midnight-node. +// Copyright (C) 2025-2026 Midnight Foundation +// SPDX-License-Identifier: Apache-2.0 +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Storage migration from v1 to v2: ledger-v8 -> ledger-v9 state translation. +//! +//! This is the on-chain half of the ledger 8 -> 9 hardfork. The pallet stores +//! only the ledger state's arena root in [`crate::StateKey`]; the `LedgerState` +//! itself lives in the ledger arena (parity-db). This migration hands the v8 +//! root to the [`migrate_state_v8_to_v9`](midnight_node_ledger::host_api::migration_8_to_9) +//! host function, which walks the v8 state, translates it into the v9 shape +//! (see [`midnight_node_ledger::state_translation_v8_to_v9`]), re-persists it, +//! and returns the new v9 root — which we write back into `StateKey`. +//! +//! It is a single-block migration: the host call translates the whole state in +//! one shot (a few milliseconds for dev/undeployed-sized state). It is wired +//! into [`crate::Migrations`](../../../runtime/src/lib.rs) via +//! [`VersionedMigration`], so it runs at most once, when a runtime at +//! pallet-midnight storage version 1 upgrades to this runtime (storage version +//! 2). A chain whose genesis is already ledger-9 starts at storage version 2 +//! and never runs it. +//! +//! Storage version 1 does not, however, imply the *ledger* state is still v8: +//! the 2.0.0 runtime already ran ledger-9 yet shipped pallet-midnight at storage +//! version 1 (it had no v1->v2 migration), so a network upgrading 2.0.0 -> this +//! runtime triggers this migration over an already-v9 state. The host function +//! detects that case and no-ops (returns the `StateKey` unchanged), so the only +//! effect on that path is the storage-version bump to 2. + +#[cfg(feature = "try-runtime")] +extern crate alloc; + +use crate::{Pallet, StateKey, pallet::Config}; +use frame_support::{ + migrations::VersionedMigration, pallet_prelude::*, traits::UncheckedOnRuntimeUpgrade, +}; +use midnight_node_ledger::types::active_ledger_bridge as LedgerApi; + +#[cfg(feature = "try-runtime")] +use alloc::vec::Vec; + +/// [`UncheckedOnRuntimeUpgrade`] implementation wrapped by [`MigrateV1ToV2`]. +pub struct InnerMigrateV1ToV2(core::marker::PhantomData); + +impl UncheckedOnRuntimeUpgrade for InnerMigrateV1ToV2 { + fn on_runtime_upgrade() -> Weight { + let state_key = StateKey::::get(); + + // The host function reads the v8 arena root, translates the referenced + // `LedgerState` to v9, re-persists it, and returns the new v9 root + // together with the synthetic cost (picoseconds) it charged the + // translation against the ledger's deterministic cost model. If the + // state is already v9 (e.g. a 2.0.0 -> this-runtime upgrade), it no-ops + // and returns the `state_key` unchanged with zero cost. + // A genuine failure here is unrecoverable: the chain would be left + // pointing at a v8 state a ledger-9 runtime cannot read, so we abort the + // upgrade. + let (new_state_key, consumed_cost_ps) = LedgerApi::migrate_state_v8_to_v9(&state_key) + .expect("FATAL: ledger v8->v9 state migration failed"); + + StateKey::::put(new_state_key); + log::info!( + target: "midnight::migration", + "ledger v8->v9 state migration complete; StateKey re-pointed to v9 root ({consumed_cost_ps}ps synthetic cost)" + ); + + // The translation runs natively in the host call, so the pallet-side + // read/write above is negligible next to it; report the ledger's own + // synthetic cost as `ref_time` instead. This is the same 1:1 mapping + // `pallet_midnight::get_tx_weight` uses for ordinary transactions + // (ledger picoseconds -> `Weight` ref_time), and that cost model + // already prices the arena reads/writes the translation performs, so + // no separate `DbWeight` charge is added on top. `proof_size` is 0: + // this chain doesn't build a PoV. Capped at the block's max weight, + // since a pathological state could in principle exceed it. + Weight::from_parts(consumed_cost_ps, 0).min(T::BlockWeights::get().max_block) + } + + #[cfg(feature = "try-runtime")] + fn pre_upgrade() -> Result, sp_runtime::TryRuntimeError> { + frame_support::ensure!( + !StateKey::::get().is_empty(), + "ledger StateKey must be populated before v8->v9 migration" + ); + Ok(Vec::new()) + } + + #[cfg(feature = "try-runtime")] + fn post_upgrade(_state: Vec) -> Result<(), sp_runtime::TryRuntimeError> { + frame_support::ensure!( + !StateKey::::get().is_empty(), + "ledger StateKey must remain populated after v8->v9 migration" + ); + Ok(()) + } +} + +/// Translates the ledger state from v8 to v9 and bumps pallet-midnight storage +/// version 1 -> 2. Wired into the runtime `Migrations` tuple. +pub type MigrateV1ToV2 = VersionedMigration< + 1, + 2, + InnerMigrateV1ToV2, + Pallet, + ::DbWeight, +>; diff --git a/runtime/src/lib.rs b/runtime/src/lib.rs index 5542e4c17..550e220df 100644 --- a/runtime/src/lib.rs +++ b/runtime/src/lib.rs @@ -280,7 +280,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { // The version of the runtime specification. A full node will not attempt to use its native // runtime in substitute for the on-chain Wasm runtime unless all of `spec_name`, // `spec_version`, and `authoring_version` are the same between Wasm and native. - spec_version: 002_000_000, + spec_version: 002_001_000, impl_version: 0, apis: RUNTIME_API_VERSIONS, transaction_version: 4, @@ -1119,7 +1119,13 @@ pub type Executive = frame_executive::Executive< /// Extrinsic type that has already been checked. pub type CheckedExtrinsic = generic::CheckedExtrinsic; /// Migrations to apply on runtime upgrade. -pub type Migrations = (pallet_throttle::migrations::v1::MigrateV0ToV1,); +pub type Migrations = ( + pallet_throttle::migrations::v1::MigrateV0ToV1, + // Ledger v8 -> v9 state translation (the ledger 8->9 hardfork). Runs once, + // when a ledger-8 runtime (pallet-midnight storage version 1) upgrades to + // this ledger-9 runtime (storage version 2). + pallet_midnight::migrations::v2::MigrateV1ToV2, +); impl frame_system::offchain::CreateTransaction for Runtime where diff --git a/util/toolkit-js/package-lock.json b/util/toolkit-js/package-lock.json index fb2ecb34a..d5e5ae874 100644 --- a/util/toolkit-js/package-lock.json +++ b/util/toolkit-js/package-lock.json @@ -2880,9 +2880,9 @@ "license": "MIT" }, "node_modules/nanoid": { - "version": "3.3.12", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.12.tgz", - "integrity": "sha512-ZB9RH/39qpq5Vu6Y+NmUaFhQR6pp+M2Xt76XBnEwDaGcVAqhlvxrl3B2bKS5D3NH3QR76v3aSrKaF/Kiy7lEtQ==", + "version": "3.3.18", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.18.tgz", + "integrity": "sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==", "dev": true, "funding": [ { @@ -3026,9 +3026,9 @@ } }, "node_modules/postcss": { - "version": "8.5.15", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.15.tgz", - "integrity": "sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A==", + "version": "8.5.26", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.26.tgz", + "integrity": "sha512-u82N74LFzG8ca+dD8puPnplTXoGH4fTPpVGuIbt36G3qvNlkvfD0lEAZSxaly3KX8TS/L1A1gsCEmvKmBcVbkQ==", "dev": true, "funding": [ { @@ -3046,7 +3046,7 @@ ], "license": "MIT", "dependencies": { - "nanoid": "^3.3.12", + "nanoid": "^3.3.17", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" }, @@ -3505,9 +3505,9 @@ } }, "node_modules/undici": { - "version": "7.28.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-7.28.0.tgz", - "integrity": "sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA==", + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.29.0.tgz", + "integrity": "sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==", "license": "MIT", "engines": { "node": ">=20.18.1" @@ -3809,34 +3809,6 @@ "engines": { "node": ">=6" } - }, - "v7": { - "name": "@midnight-ntwrk/node-toolkit-v7", - "version": "0.1.0", - "extraneous": true, - "license": "Apache-2.0", - "dependencies": { - "@effect/cli": "^0.73.2", - "@effect/platform-node": "^0.104.1", - "@midnight-ntwrk/compact-js": "2.4.3", - "@midnight-ntwrk/compact-js-command": "2.4.3", - "@midnight-ntwrk/compact-js-node": "2.4.3", - "effect": "^3.21.0" - } - }, - "v8": { - "name": "@midnight-ntwrk/node-toolkit-v8", - "version": "0.1.1", - "extraneous": true, - "license": "Apache-2.0", - "dependencies": { - "@effect/cli": "^0.74.0", - "@effect/platform-node": "^0.105.0", - "@midnight-ntwrk/compact-js": "2.5.1", - "@midnight-ntwrk/compact-js-command": "2.5.1", - "@midnight-ntwrk/compact-js-node": "2.5.1", - "effect": "^3.21.0" - } } } } diff --git a/util/toolkit/src/commands/runtime_upgrade.rs b/util/toolkit/src/commands/runtime_upgrade.rs index 2d145167f..367b6daf2 100644 --- a/util/toolkit/src/commands/runtime_upgrade.rs +++ b/util/toolkit/src/commands/runtime_upgrade.rs @@ -14,9 +14,13 @@ // limitations under the License. use std::str::FromStr; +use std::time::Duration; use clap::Args; -use subxt::{OnlineClient, SubstrateConfig, dynamic}; +use subxt::{ + OnlineClient, SubstrateConfig, dynamic, + rpcs::{RpcClient, rpc_params}, +}; use thiserror::Error; use crate::commands::root_call::{self, RootCallArgs}; @@ -35,6 +39,10 @@ pub enum RuntimeUpgradeError { ExtrinsicError(#[from] subxt::error::ExtrinsicError), #[error("transaction finalized error: {0}")] TransactionFinalizedError(#[from] subxt::error::TransactionFinalizedSuccessError), + #[error("transaction progress error: {0}")] + TransactionProgressError(#[from] subxt::error::TransactionProgressError), + #[error("rpc error: {0}")] + RpcError(#[from] subxt::rpcs::Error), #[error("events error: {0}")] EventsError(#[from] subxt::error::EventsError), #[error("keypair parse error: {0}")] @@ -43,6 +51,41 @@ pub enum RuntimeUpgradeError { RootCallError(Box), #[error("runtime upgrade failed: CodeUpdated event not found")] CodeUpdateNotFound, + #[error("timed out waiting for the apply_authorized_upgrade transaction to finalize")] + ApplyFinalizeTimeout, + #[error("runtime upgrade did not enact: spec_version stayed at {0} after applying the code")] + UpgradeNotEnacted(u32), +} + +/// Query the on-chain runtime spec_version via the raw `state_getRuntimeVersion` RPC. +/// +/// We avoid subxt's typed metadata here on purpose: across a runtime upgrade the +/// client's metadata switches to the new runtime, so decoding anything encoded by +/// the old runtime (e.g. the `System.CodeUpdated` event in the apply block) is +/// unreliable. The raw JSON spec_version has no such dependency. +/// (finalized_height, spec_version at the finalized head). +/// +/// We track both because `state_getRuntimeVersion(finalized)` reports the code +/// *stored at* that block — which flips to the new runtime already at the +/// `apply_authorized_upgrade` block, even though that block *executed* under the +/// old runtime (its `MNSV` digest — how the toolkit fetcher classifies a block's +/// ledger version — is still the old spec). The first block that actually +/// executes the new runtime is `apply + 1`. So "spec flipped at the finalized +/// head" is one block early; callers must additionally wait for the finalized +/// height to advance past that point before the fetcher will see a ledger-9 block. +async fn finalized_state(rpc: &RpcClient) -> Result<(u64, u32), RuntimeUpgradeError> { + let hash: serde_json::Value = rpc.request("chain_getFinalizedHead", rpc_params![]).await?; + let header: serde_json::Value = + rpc.request("chain_getHeader", rpc_params![hash.clone()]).await?; + let version: serde_json::Value = + rpc.request("state_getRuntimeVersion", rpc_params![hash]).await?; + let height = header + .get("number") + .and_then(|n| n.as_str()) + .and_then(|s| u64::from_str_radix(s.trim_start_matches("0x"), 16).ok()) + .unwrap_or(0); + let spec = version.get("specVersion").and_then(|v| v.as_u64()).unwrap_or(0) as u32; + Ok((height, spec)) } #[derive(Args)] @@ -78,7 +121,8 @@ pub async fn execute(args: RuntimeUpgradeArgs) -> Result<(), RuntimeUpgradeError log::info!("Code hash: 0x{}", hex::encode(code_hash)); // Step 3: Build System::authorize_upgrade call and encode it - let api = OnlineClient::::from_insecure_url(&args.rpc_url).await?; + let rpc_client = RpcClient::from_insecure_url(&args.rpc_url).await?; + let api = OnlineClient::::from_rpc_client(rpc_client.clone()).await?; let authorize_upgrade_call = dynamic::tx("System", "authorize_upgrade", vec![dynamic::Value::from_bytes(&code_hash)]); let encoded_call = api.tx().await?.call_data(&authorize_upgrade_call)?; @@ -101,28 +145,54 @@ pub async fn execute(args: RuntimeUpgradeArgs) -> Result<(), RuntimeUpgradeError let apply_upgrade_call = dynamic::tx("System", "apply_authorized_upgrade", vec![dynamic::Value::from_bytes(&code)]); - let apply_events = api - .tx() - .await? - .sign_and_submit_then_watch_default(&apply_upgrade_call, &signer) - .await? - .wait_for_finalized_success() - .await?; - - // Step 6: Verify CodeUpdated event - let mut success = false; - for event in apply_events.iter() { - let event = event?; - if event.pallet_name() == "System" && event.event_name() == "CodeUpdated" { - log::info!("Code update success: {:?}", event); - success = true; - break; + let (_, pre_spec_version) = finalized_state(&rpc_client).await?; + log::info!("Pre-upgrade spec_version (finalized): {pre_spec_version}"); + + // Wait only for finalization — NOT `wait_for_finalized_success`, which eagerly + // decodes the block's events. `apply_authorized_upgrade` swaps the on-chain + // code, and subxt's metadata follows it to the new runtime, so decoding the + // old runtime's `System.CodeUpdated` event in that block fails. Confirm the + // upgrade succeeded by observing the spec_version bump below instead. + let submit = async { + api.tx() + .await? + .sign_and_submit_then_watch_default(&apply_upgrade_call, &signer) + .await? + .wait_for_finalized() + .await + .map_err(RuntimeUpgradeError::from) + }; + tokio::time::timeout(Duration::from_secs(120), submit) + .await + .map_err(|_| RuntimeUpgradeError::ApplyFinalizeTimeout)??; + + // Step 6: Confirm the upgrade is not just applied but *executing* at a + // finalized block. `state_getRuntimeVersion(finalized)` reports the stored + // code, which flips to the new runtime already at the apply block — but that + // block's `MNSV` execution-version digest (how the fetcher classifies it) is + // still the old spec. The first block that runs the new runtime is apply+1. + // So: note the finalized height where the stored spec first exceeds pre, then + // wait for the finalized height to advance past it (apply+1 finalized). Only + // then will a downstream `fetch` see a ledger-9-classified block. + let mut flip_height: Option = None; + for _ in 0..60 { + tokio::time::sleep(Duration::from_secs(3)).await; + let (height, spec) = finalized_state(&rpc_client).await?; + if spec > pre_spec_version { + match flip_height { + None => flip_height = Some(height), + Some(h0) if height > h0 => { + log::info!( + "Runtime upgrade completed successfully! spec_version {pre_spec_version} -> {spec}; \ + new runtime executing since finalized #{}, now finalized #{height}", + h0 + 1, + ); + return Ok(()); + }, + _ => {}, + } } } - if !success { - return Err(RuntimeUpgradeError::CodeUpdateNotFound); - } - log::info!("Runtime upgrade completed successfully!"); - Ok(()) + Err(RuntimeUpgradeError::UpgradeNotEnacted(pre_spec_version)) } diff --git a/util/toolkit/src/fetcher/compute_task.rs b/util/toolkit/src/fetcher/compute_task.rs index ba09a1d3d..508525ed9 100644 --- a/util/toolkit/src/fetcher/compute_task.rs +++ b/util/toolkit/src/fetcher/compute_task.rs @@ -25,7 +25,8 @@ use crate::{ fetch_storage::{FetchStorage, FetchedBlock}, runtimes::{ MidnightMetadata, MidnightMetadata0_21_0, MidnightMetadata0_22_0, - MidnightMetadata1_0_0, MidnightMetadata2_0_0, RuntimeVersion, RuntimeVersionError, + MidnightMetadata1_0_0, MidnightMetadata2_0_0, MidnightMetadata2_1_0, RuntimeVersion, + RuntimeVersionError, }, }, }; @@ -173,6 +174,14 @@ impl ComputeTask { ) .await }, + RuntimeVersion::V2_1_0 => { + Self::process_block_with_protocol::( + block, + &header, + spec_version, + ) + .await + }, } } diff --git a/util/toolkit/src/fetcher/runtimes.rs b/util/toolkit/src/fetcher/runtimes.rs index 9eae8da2c..ec4f6170a 100644 --- a/util/toolkit/src/fetcher/runtimes.rs +++ b/util/toolkit/src/fetcher/runtimes.rs @@ -26,6 +26,7 @@ pub enum RuntimeVersion { V0_22_0, V1_0_0, V2_0_0, + V2_1_0, } impl TryFrom for RuntimeVersion { type Error = RuntimeVersionError; @@ -35,6 +36,7 @@ impl TryFrom for RuntimeVersion { 000_022_000 => Ok(Self::V0_22_0), 001_000_000 => Ok(Self::V1_0_0), 002_000_000 => Ok(Self::V2_0_0), + 002_001_000 => Ok(Self::V2_1_0), _ => Err(RuntimeVersionError::UnsupportedBlockVersion(value)), } } @@ -48,6 +50,7 @@ impl RuntimeVersion { Self::V0_22_0 => 000_022_000, Self::V1_0_0 => 001_000_000, Self::V2_0_0 => 002_000_000, + Self::V2_1_0 => 002_001_000, } } @@ -157,3 +160,9 @@ impl_midnight_metadata!( mn_meta_2_0_0, midnight_node_metadata::midnight_metadata_2_0_0 ); + +impl_midnight_metadata!( + MidnightMetadata2_1_0, + mn_meta_2_1_0, + midnight_node_metadata::midnight_metadata_2_1_0 +); diff --git a/util/toolkit/src/tx_generator/builder/mod.rs b/util/toolkit/src/tx_generator/builder/mod.rs index 9ce2f9558..60161154e 100644 --- a/util/toolkit/src/tx_generator/builder/mod.rs +++ b/util/toolkit/src/tx_generator/builder/mod.rs @@ -19,7 +19,7 @@ use midnight_node_ledger_helpers::fork::{ fork_aware_context::{ ForkAwareLedgerContext, apply_block_7, apply_block_8, apply_block_9, block_context_from_raw_7, block_context_from_raw_8, block_context_from_raw_9, - fork_context_7_to_8, + fork_context_7_to_8, fork_context_8_to_9, }, raw_block_data::{LedgerVersion, RawBlockData}, }; @@ -1110,6 +1110,24 @@ fn replay_blocks_9( } } +/// Fork a ledger-8 context to ledger 9 (real state translation) and replay the +/// ledger-9 blocks, if any. Returns the ledger-8 context unchanged when there are +/// no ledger-9 blocks. +fn fork_8_to_9_if_needed( + ctx8: midnight_node_ledger_helpers::ledger_8::context::LedgerContext, + l9_blocks: &[RawBlockData], + cached: &[(WalletSeed, CachedWalletState)], +) -> ForkAwareLedgerContext { + if l9_blocks.is_empty() { + ForkAwareLedgerContext::Ledger8(ctx8) + } else { + let ctx9 = + timed!("fork_context_8_to_9", fork_context_8_to_9(ctx8)).expect("fork 8 to 9 failed"); + replay_blocks_9(&ctx9, l9_blocks, cached); + ForkAwareLedgerContext::Ledger9(ctx9) + } +} + /// Replays blocks across a potential Ledger7→Ledger8->Ledger9 fork boundaries, /// injecting cached wallets at their saved height. pub(crate) fn replay_blocks( @@ -1133,27 +1151,27 @@ pub(crate) fn replay_blocks( l8_and_l9_blocks.partition_point(|b| b.ledger_version() == LedgerVersion::Ledger8); let (l8_blocks, l9_blocks) = l8_and_l9_blocks.split_at(fork_8_to_9_idx); - assert!( - l9_blocks.is_empty() || (l7_blocks.is_empty() && l8_blocks.is_empty()), - "chain has Ledger9 blocks and eariler version blocks. This is not supported yet!" - ); - + // Replay each version's blocks in order, forking the context across the + // 7->8 and 8->9 boundaries as needed. The 8->9 fork performs a real state + // translation (see `fork_context_8_to_9`) so post-hardfork transactions are + // built at ledger 9, matching the upgraded chain. let result = match fork_ctx { ForkAwareLedgerContext::Ledger7(ctx7) => { replay_blocks_7(&ctx7, l7_blocks); - if l8_blocks.is_empty() { - assert!(cached.is_empty(), "cached wallets with no Ledger8 blocks"); + if l8_blocks.is_empty() && l9_blocks.is_empty() { + assert!(cached.is_empty(), "cached wallets with no Ledger8/9 blocks"); ForkAwareLedgerContext::Ledger7(ctx7) } else { - let ctx8 = fork_context_7_to_8(ctx7).expect("fork 7 to 8 failed"); + let ctx8 = timed!("fork_context_7_to_8", fork_context_7_to_8(ctx7)) + .expect("fork 7 to 8 failed"); replay_blocks_8(&ctx8, l8_blocks); - ForkAwareLedgerContext::Ledger8(ctx8) + fork_8_to_9_if_needed(ctx8, l9_blocks, cached) } }, ForkAwareLedgerContext::Ledger8(ctx8) => { assert!(l7_blocks.is_empty(), "Ledger7 blocks with Ledger8 context"); replay_blocks_8(&ctx8, l8_blocks); - ForkAwareLedgerContext::Ledger8(ctx8) + fork_8_to_9_if_needed(ctx8, l9_blocks, cached) }, ForkAwareLedgerContext::Ledger9(ctx9) => { assert!(l7_blocks.is_empty(), "Ledger7 blocks with Ledger9 context"); diff --git a/util/toolkit/test-images.docker-compose.yml b/util/toolkit/test-images.docker-compose.yml index 42cdd834f..71a079559 100644 --- a/util/toolkit/test-images.docker-compose.yml +++ b/util/toolkit/test-images.docker-compose.yml @@ -15,8 +15,11 @@ services: depends_on: guard: condition: service_completed_successfully + # Ledger-8 release: the hardfork_e2e test forks from this (ledger 8, runtime + # spec_version 1_000_000, pallet-midnight storage version 1) up to the current + # ledger-9 runtime, exercising the v8->v9 state migration. midnight-node-fork-from: - image: ${FORK_FROM_NODE_IMAGE:-midnightntwrk/midnight-node:0.21.0} + image: ${FORK_FROM_NODE_IMAGE:-midnightntwrk/midnight-node:1.0.1} depends_on: guard: condition: service_completed_successfully diff --git a/util/toolkit/tests/hardfork_e2e.rs b/util/toolkit/tests/hardfork_e2e.rs index 578788807..da7d11a63 100644 --- a/util/toolkit/tests/hardfork_e2e.rs +++ b/util/toolkit/tests/hardfork_e2e.rs @@ -19,6 +19,7 @@ use clap::Parser; use common::{test_image, wait_for_node::wait_for_finalized_block}; use midnight_node_toolkit::cli::{Cli, run_command}; use std::{process::Command, time::Duration}; +use subxt::rpcs::{RpcClient, rpc_params}; use testcontainers::{ GenericImage, ImageExt, core::{ContainerPort, WaitFor}, @@ -53,8 +54,112 @@ async fn run_cli(args: &[&str]) { eprintln!("[hardfork_e2e] CLI command succeeded"); } +/// Hash of the block at `height`, hex-encoded. +async fn block_hash_at(rpc: &RpcClient, height: u64) -> String { + let hash: serde_json::Value = rpc + .request("chain_getBlockHash", rpc_params![height]) + .await + .unwrap_or_else(|e| panic!("chain_getBlockHash({height}) failed: {e}")); + hash.as_str() + .unwrap_or_else(|| panic!("no block at height {height}")) + .to_owned() +} + +/// The runtime `specVersion` *stored at* `hash`. +/// +/// Raw `state_getRuntimeVersion` rather than subxt's typed metadata on purpose: +/// across a runtime upgrade the client's metadata follows the new runtime, so +/// anything the old runtime encoded decodes unreliably. See +/// `midnight_node_toolkit::commands::runtime_upgrade`. +async fn spec_version_at(rpc: &RpcClient, hash: &str) -> u64 { + let version: serde_json::Value = rpc + .request("state_getRuntimeVersion", rpc_params![hash]) + .await + .unwrap_or_else(|e| panic!("state_getRuntimeVersion({hash}) failed: {e}")); + version + .get("specVersion") + .and_then(|v| v.as_u64()) + .unwrap_or_else(|| panic!("no specVersion in runtime version at {hash}")) +} + +async fn finalized_height(rpc: &RpcClient) -> u64 { + let hash: serde_json::Value = rpc + .request("chain_getFinalizedHead", rpc_params![]) + .await + .expect("chain_getFinalizedHead failed"); + let header: serde_json::Value = rpc + .request("chain_getHeader", rpc_params![hash]) + .await + .expect("chain_getHeader failed"); + header + .get("number") + .and_then(|n| n.as_str()) + .and_then(|s| u64::from_str_radix(s.trim_start_matches("0x"), 16).ok()) + .expect("no number in finalized header") +} + +/// Locate the block that applied the new runtime code — the one whose committed +/// state pairs the *new* `:code` with the *old* ledger version's `StateKey`. +/// +/// `frame_system` overwrites `:code` inside that block (the pre-fork runtime ships +/// `system_version: 1`, so the code is not staged in `:pending_code`), while +/// pallet-midnight's v8->v9 state translation only runs in the next block's +/// `initialize_block`. Executing a read at that hash therefore runs ledger-9 WASM +/// against a ledger-8 arena root, which is what GH #1959 reports. +/// +/// `state_getRuntimeVersion` reports the code stored at a block, so the first +/// height reporting the new spec is exactly that block. spec_version is monotonic +/// along the chain, so binary-search for it. +async fn find_code_applied_block(rpc: &RpcClient, head: u64, old_spec: u64) -> u64 { + let (mut lo, mut hi) = (1u64, head); + while lo < hi { + let mid = lo + (hi - lo) / 2; + let hash = block_hash_at(rpc, mid).await; + if spec_version_at(rpc, &hash).await > old_spec { + hi = mid; + } else { + lo = mid + 1; + } + } + lo +} + +/// Every way of reading the ledger state must answer at `height`. +/// +/// Both the `midnight_*` RPCs and a raw `state_call`: the fix lives in the ledger-9 +/// host function, so the runtime API has to work at the skew block too — that is +/// the path subxt-based tooling (`chain-indexer`, GH #1969) takes, and it does not +/// go anywhere near the node's own RPC layer. +async fn assert_ledger_state_readable(rpc: &RpcClient, height: u64, label: &str) { + let hash = block_hash_at(rpc, height).await; + + for method in ["midnight_zswapStateRoot", "midnight_ledgerStateRoot"] { + let root: Vec = rpc + .request(method, rpc_params![&hash]) + .await + .unwrap_or_else(|e| panic!("{method} failed at {label} (#{height}, {hash}): {e}")); + assert!(!root.is_empty(), "{method} returned an empty root at {label} (#{height})"); + } + + // `Result, LedgerApiError>` SCALE-encoded: a leading 0x00 is `Ok`, and + // anything else is the pallet reporting a ledger error (0x01 plus the variant). + for api in + ["MidnightRuntimeApi_get_ledger_state_root", "MidnightRuntimeApi_get_ledger_parameters"] + { + let encoded: String = rpc + .request("state_call", rpc_params![api, "0x", &hash]) + .await + .unwrap_or_else(|e| panic!("{api} failed at {label} (#{height}, {hash}): {e}")); + assert!( + encoded.starts_with("0x00"), + "{api} returned an error at {label} (#{height}): {encoded}" + ); + } + + eprintln!("[hardfork_e2e] ledger state readable at {label} (#{height})"); +} + #[test_log::test(tokio::test)] -#[ignore = "Migration to Ledger v9 is not yet supported. Issue #1580."] async fn hardfork_single_tx() { // 1. Generate chain-spec from fork-from node let (old_name, old_tag) = test_image("midnight-node-fork-from"); @@ -141,7 +246,34 @@ async fn hardfork_single_tx() { ]) .await; - // 5. Post-fork: run single-tx again to verify the node still works after the (future) upgrade + // 5. GH #1959: the whole fork boundary must stay readable. The block that + // applied the new code carries a ledger-8 `StateKey` under ledger-9 `:code`, + // so the ledger-9 host API has to detect that and serve the read from the + // ledger-8 bridge; the block after it is already translated to v9 and must + // keep taking the ordinary ledger-9 path. + let rpc = RpcClient::from_insecure_url(&url).await.expect("failed to open raw RPC client"); + let pre_fork_spec = { + let hash = block_hash_at(&rpc, 1).await; + spec_version_at(&rpc, &hash).await + }; + let head = finalized_height(&rpc).await; + let applied = find_code_applied_block(&rpc, head, pre_fork_spec).await; + eprintln!( + "[hardfork_e2e] new runtime code applied at #{applied} \ + (pre-fork spec {pre_fork_spec}, finalized head #{head})" + ); + assert!(applied > 1, "expected the code-applying block to be past #1, got #{applied}"); + assert!(applied < head, "expected the code-applying block to be below the finalized head"); + + // `runtime-upgrade` already waits for finality to pass `applied`, but the + // assertion below needs `applied + 1` to exist regardless. + wait_for_finalized_block(&url, applied + 1, Duration::from_secs(60)).await; + + assert_ledger_state_readable(&rpc, applied - 1, "pre-fork").await; + assert_ledger_state_readable(&rpc, applied, "code-applied block").await; + assert_ledger_state_readable(&rpc, applied + 1, "post-migration").await; + + // 6. Post-fork: run single-tx again to verify the node still works after the (future) upgrade run_cli(&[ "generate-txs", "--fetch-cache",