Skip to content

Copy pinned TigerBeetle source without ledger state machine #2

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
gpu004 merged 1 commit into from
Jul 13, 2026
Merged

Copy pinned TigerBeetle source without ledger state machine #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
100 changes: 100 additions & 0 deletions ocam/.github/ci/test_aof.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
#!/usr/bin/env bash
set -eEuo pipefail

# Download Zig if it does not yet exist:
if [ ! -f "zig/zig" ]; then
./zig/download.sh
fi

./zig/zig build install -Drelease

# Be careful to use a benchmark-specific filenames so that we don't erase a real data file:
cleanup() {
rm -f aof-test.tigerbeetle
rm -f aof-test.tigerbeetle.aof
rm -f aof.log
rm -f {a1,a2,b1,b2}/aof-test.tigerbeetle{,.aof}
rmdir {a1,a2,b1,b2} 2>/dev/null || true
}
cleanup

function onerror {
if [ "$?" == "0" ]; then
cleanup
else
echo
echo "============================================================="
echo "Error running aof test, here are more details (from aof.log):"
echo "============================================================="
cat aof.log
fi

kill $(jobs -p) 2> /dev/null || true
wait
}
trap onerror EXIT

echo "Running benchmark to populate AOF..."
./tigerbeetle format --cluster=0 --replica=0 --replica-count=1 aof-test.tigerbeetle > aof.log 2>&1
./tigerbeetle start --cache-grid=256MiB --addresses=3000 --aof --experimental aof-test.tigerbeetle >> aof.log 2>&1 &
./tigerbeetle benchmark --addresses=3000 --transfer-count=400000 >> aof.log 2>&1
kill %1

echo ""
echo "Running 'zig build aof -- debug aof-test.tigerbeetle.aof' to check AOF..."
data_checksum_src=$(./zig/zig build aof -- debug aof-test.tigerbeetle.aof 2>&1 | tee -a aof.log | grep 'Data checksum chain:')
echo "${data_checksum_src}"

mkdir a1 a2 b1 b2

echo ''
echo 'Testing recovery...'
./tigerbeetle format --cluster=0 --replica=0 --replica-count=2 a1/aof-test.tigerbeetle >> ./aof.log 2>&1
./tigerbeetle start --aof-recovery --cache-grid=256MiB --addresses=3001,3002 --aof-file=a1/aof-test.tigerbeetle.aof --experimental a1/aof-test.tigerbeetle >> ./aof.log 2>&1 &
r1=$!
./tigerbeetle format --cluster=0 --replica=1 --replica-count=2 a2/aof-test.tigerbeetle >> ./aof.log 2>&1
./tigerbeetle start --aof-recovery --cache-grid=256MiB --addresses=3001,3002 --aof --experimental a2/aof-test.tigerbeetle >> ./aof.log 2>&1 &
r2=$!

sleep 1
./zig/zig build aof -- recover --cluster=0 --addresses=3001,3002 aof-test.tigerbeetle.aof >> aof.log 2>&1
sleep 10 # Give replicas time to settle.
kill $r1 $r2

echo ""
echo "Recovering a second time, to test determinism."

./tigerbeetle format --cluster=0 --replica=0 --replica-count=2 b1/aof-test.tigerbeetle >> ./aof.log 2>&1
./tigerbeetle start --aof-recovery --cache-grid=256MiB --addresses=3001,3002 --experimental b1/aof-test.tigerbeetle >> ./aof.log 2>&1 &
r1=$!
./tigerbeetle format --cluster=0 --replica=1 --replica-count=2 b2/aof-test.tigerbeetle >> ./aof.log 2>&1
./tigerbeetle start --aof-recovery --cache-grid=256MiB --addresses=3001,3002 --experimental b2/aof-test.tigerbeetle >> ./aof.log 2>&1 &
r2=$!

./zig/zig build aof -- recover --cluster=0 --addresses=3001,3002 aof-test.tigerbeetle.aof >> aof.log 2>&1
sleep 10 # Give replicas time to settle.
kill $r1 $r2

echo ""
echo "Running 'zig build aof -- debug a{1,2}/aof-test.tigerbeetle.aof' to check recovered AOF..."
data_checksum_recovered_1=$(./zig/zig build aof -- debug a1/aof-test.tigerbeetle.aof 2>&1 | tee -a aof.log | grep 'Data checksum chain:')
echo "1: ${data_checksum_recovered_1}"
data_checksum_recovered_2=$(./zig/zig build aof -- debug a2/aof-test.tigerbeetle.aof 2>&1 | tee -a aof.log | grep 'Data checksum chain:')
echo "2: ${data_checksum_recovered_2}"

if [ "${data_checksum_src}" != "${data_checksum_recovered_1}" ] || [ "${data_checksum_src}" != "${data_checksum_recovered_2}" ]; then
echo "Mismatch in data checksums!"
exit 1
fi

echo
echo 'Running "tigerbeetle inspect" to compare superblocks...'
superblock_a=$(./tigerbeetle inspect superblock ./a1/aof-test.tigerbeetle 2>/dev/null)
superblock_b=$(./tigerbeetle inspect superblock ./b1/aof-test.tigerbeetle 2>/dev/null)
if [ "$superblock_a" != "$superblock_b" ]; then
echo "Mismatch in recovery determinism."
exit 1
fi

echo
echo 'Success!'
228 changes: 228 additions & 0 deletions ocam/.github/workflows/ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,228 @@
name: CI
permissions: {}

concurrency:
group: core-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}

on:
merge_group:
pull_request:
push:
branches: ["main"]

env:
GH_TOKEN: ${{ github.token }}
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true # Remove once this is the default (June 2nd, 2026).

jobs:
smoke:
runs-on: ubuntu-latest
steps:
- &checkout
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with: { fetch-depth: 2147483647, fetch-tags: true } # Fetch history for "git tag" in build.zig.
- &cache
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
# actions/cache (as of v5.0.5) routinely flakes under windows -- it just prints
# "Cache hit for: Windows-X64-(hash)" and then exits with no other info.
continue-on-error: ${{ startsWith(runner.os, 'Windows') }}
with:
path: ./zig/cache
key: ${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('./zig/download.sh') }}
- run: shellcheck ./zig/download.sh
- shell: pwsh
run: Invoke-ScriptAnalyzer -Path zig/download.win.ps1 -Severity Error,Warning,Information -EnableExit
- run: ./zig/download.ps1 && ./zig/zig build --summary all ci -- smoke

test:
strategy:
matrix:
include:
- { os: 'ubuntu-latest' }
- { os: 'ubuntu-latest-arm64' }
- { os: 'windows-latest' }
- { os: 'macos-latest' }
- { os: 'macos-15-intel' }
runs-on: ${{ matrix.os }}
steps:
- run: git config --global core.autocrlf false
- if: matrix.os == 'ubuntu-latest' || matrix.os == 'ubuntu-latest-arm64'
run: | # Allow unshare for vortex.
sudo sysctl -w kernel.apparmor_restrict_unprivileged_unconfined=0
sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
- *checkout
- *cache
- &select_xcode
# TODO(Zig): Xcode >26.3 breaks linking with Zig 0.14.1.
# Pin to Xcode 26.3 when it's available. Remove once we upgrade Zig.
# See https://codeberg.org/ziglang/zig/issues/31658
if: matrix.os == 'macos-latest'
run: |
if [ -d /Applications/Xcode_26.3.app ]; then
sudo xcode-select -s /Applications/Xcode_26.3.app
fi
xcodebuild -version
- run: ./zig/download.ps1 && ./zig/zig build --summary all ci -- test

test_aof:
runs-on: ubuntu-latest
steps:
- *checkout
- *cache
- run: ./zig/download.ps1 && ./zig/zig build --summary all ci -- aof

clients:
strategy:
matrix:
include:
- { os: 'ubuntu-latest', language: 'dotnet', language_version: '8.0.x' }
- { os: 'ubuntu-latest', language: 'go', language_version: '1.21' }
- { os: 'ubuntu-latest', language: 'rust', language_version: '1.71' }
- { os: 'ubuntu-latest', language: 'rust', language_version: 'stable' }
- { os: 'ubuntu-latest', language: 'java', language_version: '11' }
- { os: 'ubuntu-latest', language: 'java', language_version: '21' }
- { os: 'ubuntu-latest', language: 'node', language_version: '18.x' }
- { os: 'ubuntu-latest', language: 'node', language_version: '24.x' }
- { os: 'ubuntu-latest', language: 'ruby', language_version: '3.3' }
- { os: 'ubuntu-latest', language: 'ruby', language_version: '4.0' }

# Support Python 3.7 explicitly, even though it's EOL.
- { os: 'ubuntu-22.04', language: 'python', language_version: '3.7' }
- { os: 'ubuntu-latest', language: 'python', language_version: '3.13' }

- { os: 'windows-latest', language: 'dotnet', language_version: '8.0.x' }
- { os: 'windows-latest', language: 'go', language_version: '1.21' }
- { os: 'windows-latest', language: 'rust', language_version: '1.71' }
- { os: 'windows-latest', language: 'rust', language_version: 'stable' }
- { os: 'windows-latest', language: 'java', language_version: '11' }
- { os: 'windows-latest', language: 'java', language_version: '21' }
- { os: 'windows-latest', language: 'node', language_version: '18.x' }
- { os: 'windows-latest', language: 'node', language_version: '20.x' }
- { os: 'windows-latest', language: 'python', language_version: '3.7' }
- { os: 'windows-latest', language: 'python', language_version: '3.13' }
- { os: 'windows-latest', language: 'ruby', language_version: '3.3' }
- { os: 'windows-latest', language: 'ruby', language_version: '4.0' }

# Limited matrix for macOS - runners are concurrency limited.
- { os: 'macos-latest', language: 'go', language_version: '1.21' }
- { os: 'macos-latest', language: 'node', language_version: '20.x' }
- { os: 'macos-latest', language: 'python', language_version: '3.13' }
- { os: 'macos-latest', language: 'ruby', language_version: '4.0' }

- { os: 'macos-15-intel', language: 'go', language_version: '1.21' }
- { os: 'macos-15-intel', language: 'node', language_version: '20.x' }
- { os: 'macos-15-intel', language: 'python', language_version: '3.13' }
- { os: 'macos-15-intel', language: 'ruby', language_version: '4.0' }

# Limited matrix for Ubuntu ARM - runners are paid and we're not sure of the cost yet.
- { os: 'ubuntu-latest-arm64', language: 'dotnet', language_version: '8.0.x' }
- { os: 'ubuntu-latest-arm64', language: 'go', language_version: '1.21' }
- { os: 'ubuntu-latest-arm64', language: 'java', language_version: '21' }
- { os: 'ubuntu-latest-arm64', language: 'node', language_version: '20.x' }
- { os: 'ubuntu-latest-arm64', language: 'python', language_version: '3.13' }
- { os: 'ubuntu-latest-arm64', language: 'ruby', language_version: '4.0' }

runs-on: ${{ matrix.os }}
steps:
- run: git config --global core.autocrlf false
- *checkout
- *cache
- if: matrix.os == 'ubuntu-latest' || matrix.os == 'ubuntu-latest-arm64'
run: | # Allow unshare for vortex.
sudo sysctl -w kernel.apparmor_restrict_unprivileged_unconfined=0
sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
- *select_xcode
- if: matrix.language == 'dotnet'
uses: actions/setup-dotnet@c2fa09f4bde5ebb9d1777cf28262a3eb3db3ced7 # v5.2.0
with: { dotnet-version: "${{ matrix.language_version }}" }

- if: matrix.language == 'go'
uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
with: { go-version: "${{ matrix.language_version }}" }

- if: matrix.language == 'rust'
run: rustup default ${{ matrix.language_version }} && rustup component add clippy rustfmt

- if: matrix.language == 'java'
uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0
with: { java-version: "${{ matrix.language_version }}", distribution: 'temurin'}

- if: matrix.language == 'java'
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
# actions/cache (as of v5.0.5) routinely flakes under windows -- it just prints
# "Cache hit for: Windows-X64-(hash)" and then exits with no other info.
continue-on-error: ${{ startsWith(runner.os, 'Windows') }}
with:
path: ~/.m2/repository
key: setup-java-${{ runner.os }}-${{ runner.arch }}-maven-${{ hashFiles('**/pom.xml') }}

- if: matrix.language == 'node'
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
with: { node-version: "${{ matrix.language_version }}" }

- if: matrix.language == 'python'
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
with: { python-version: "${{ matrix.language_version }}" }
- if: matrix.language == 'python'
run: pip install pytest 'mypy<=1.18.2'

- if: matrix.language == 'ruby'
uses: ruby/setup-ruby@97ecb7b512899eb71ab1bf2310a624c6f1589ac6 # v1.308.0
with: { ruby-version: "${{ matrix.language_version }}" }

- run: ./zig/download.ps1 && ./zig/zig build --summary all ci -- ${{ matrix.language }}

devhub:
runs-on: ubuntu-22.04
environment: ${{ github.ref == 'refs/heads/main' && 'devhub' || '' }}
permissions:
pages: write
id-token: write

steps:
- *checkout
- *cache
- run: sudo apt-get update && sudo apt-get install -y kcov
- run: sudo rm -rf /usr/local/lib/android # Free up disk space for benchmarking.
- run: ./zig/download.ps1

# Dummy devhub run - checks that all the devhub tests pass in CI. They are run again, in main
# once merged. Kcov is skipped to avoid adding to the pipeline time.
- if: github.ref != 'refs/heads/main'
run: sudo -E ./zig/zig build --summary all ci -- devhub-dry-run
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# Not providing DEVHUBDB_PAT and NYRKIO_TOKEN stops devhub from uploading its results, but
# it still runs everything.

# Run under sudo to enable memory locking for accurate RSS stats.
- if: github.ref == 'refs/heads/main'
run: sudo -E ./zig/zig build --summary all ci -- devhub
env:
DEVHUBDB_PAT: ${{ secrets.DEVHUBDB_PAT }}
NYRKIO_TOKEN: ${{ secrets.NYRKIO_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

- if: github.ref == 'refs/heads/main'
uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0
with: { path: ./src/devhub }

- if: github.ref == 'refs/heads/main'
uses: actions/deploy-pages@cd2ce8fcbc39b97be8ca5fce6e763baed58fa128 # v5.0.0


# Work around GitHub considering Skipped jobs success for "Require status checks before merging"
# See also:
# https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/managing-protected-branches/about-protected-branches#require-status-checks-before-merging
# https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/collaborating-on-repositories-with-code-quality-features/troubleshooting-required-status-checks#handling-skipped-but-required-checks
# https://stackoverflow.com/a/75250293
core-pipeline:
if: always() && github.event_name == 'merge_group'
runs-on: ubuntu-latest
needs: [smoke, test, test_aof, clients, devhub]
steps:
- if: ${{ !(contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')) }}
run: exit 0
- if: ${{ (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')) }}
run: exit 1
Loading